Edit Book: Run syntax highlighting in the background so that it does not affect responsiveness of the user interface

Technically, the syntax highlighting now does idle processing, that is
it yields after every line of text is highlighted to allow the rest of
the UI to respond to user interaction. This gives an excellent
illusion of running int he background, except for pathological files
with very long lines (such as those that have all their markup on a
single line).

src/calibre/gui2/tweak_book/editor/smart/html.py

@@ -238,6 +238,7 @@ def add_tag(tag):
         return ans
 
     def rename_block_tag(self, editor, new_name):
+        editor.highlighter.join()
         c = editor.textCursor()
         block, offset = c.block(), c.positionInBlock()
         tag = None
@@ -268,6 +269,7 @@ def rename_block_tag(self, editor, new_name):
                 'No suitable block level tag was found to rename'), show=True)
 
     def get_smart_selection(self, editor, update=True):
+        editor.highlighter.join()
         cursor = editor.textCursor()
         if not cursor.hasSelection():
             return ''
@@ -288,6 +290,7 @@ def get_smart_selection(self, editor, update=True):
         return editor.selected_text_from_cursor(cursor)
 
     def insert_hyperlink(self, editor, target, text):
+        editor.highlighter.join()
         c = editor.textCursor()
         if c.hasSelection():
             c.insertText('')  # delete any existing selected text
@@ -301,6 +304,7 @@ def insert_hyperlink(self, editor, target, text):
         editor.setTextCursor(c)
 
     def insert_tag(self, editor, name):
+        editor.highlighter.join()
         name = name.lstrip()
         text = self.get_smart_selection(editor, update=True)
         c = editor.textCursor()
@@ -314,6 +318,7 @@ def insert_tag(self, editor, name):
     def verify_for_spellcheck(self, cursor, highlighter):
         # Return True iff the cursor is in a location where spelling is
         # checked (inside a tag or inside a checked attribute)
+        highlighter.join()
         block = cursor.block()
         start_pos = cursor.anchor() - block.position()
         end_pos = cursor.position() - block.position()
@@ -393,6 +398,7 @@ def goto_sourceline(self, editor, sourceline, tags, attribute=None):
     def get_inner_HTML(self, editor):
         ''' Select the inner HTML of the current tag. Return a cursor with the
         inner HTML selected or None. '''
+        editor.highlighter.join()
         c = editor.textCursor()
         block = c.block()
         offset = c.position() - block.position()

src/calibre/gui2/tweak_book/editor/syntax/base.py

@@ -7,10 +7,10 @@
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import sys
-from collections import defaultdict
+from collections import defaultdict, deque
 
 from PyQt4.Qt import (
-    QTextCursor, pyqtSlot, QTextBlockUserData, QTextLayout)
+    QTextCursor, pyqtSlot, QTextBlockUserData, QTextLayout, QTimer)
 
 from ..themes import highlight_to_char_format
 from calibre.gui2.tweak_book.widgets import BusyCursor
@@ -71,6 +71,12 @@ class SyntaxHighlighter(object):
 
     def __init__(self):
         self.doc = None
+        self.requests = deque()
+        self.ignore_requests = False
+
+    @property
+    def has_requests(self):
+        return bool(self.requests)
 
     def apply_theme(self, theme):
         self.theme = {k:highlight_to_char_format(v) for k, v in theme.iteritems()}
@@ -117,24 +123,77 @@ def get_user_data(self, block):
     @pyqtSlot(int, int, int)
     def reformat_blocks(self, position, removed, added):
         doc = self.doc
-        if doc is None or not hasattr(self, 'state_map'):
+        if doc is None or self.ignore_requests or not hasattr(self, 'state_map'):
             return
+
+        block = doc.findBlock(position)
+        if not block.isValid():
+            return
+        start_cursor = QTextCursor(block)
         last_block = doc.findBlock(position + added + (1 if removed > 0 else 0))
         if not last_block.isValid():
             last_block = doc.lastBlock()
-        end_pos = last_block.position() + last_block.length()
-        force_next_highlight = False
+        end_cursor = QTextCursor(last_block)
+        end_cursor.movePosition(end_cursor.EndOfBlock)
+        self.requests.append((start_cursor, end_cursor))
+        QTimer.singleShot(0, self.do_one_block)
 
-        doc.contentsChange.disconnect(self.reformat_blocks)
+    def do_one_block(self):
         try:
-            block = doc.findBlock(position)
-            while block.isValid() and (block.position() < end_pos or force_next_highlight):
-                formats, force_next_highlight = self.parse_single_block(block)
-                self.apply_format_changes(block, formats)
-                doc.markContentsDirty(block.position(), block.length())
-                block = block.next()
+            start_cursor, end_cursor = self.requests[0]
+        except IndexError:
+            return
+        self.ignore_requests = True
+        try:
+            block = start_cursor.block()
+            if not block.isValid():
+                self.requests.popleft()
+                return
+            formats, force_next_highlight = self.parse_single_block(block)
+            self.apply_format_changes(block, formats)
+            try:
+                self.doc.markContentsDirty(block.position(), block.length())
+            except AttributeError:
+                self.requests.clear()
+                return
+            ok = start_cursor.movePosition(start_cursor.NextBlock)
+            if not ok:
+                self.requests.popleft()
+                return
+            next_block = start_cursor.block()
+            if next_block.position() > end_cursor.position():
+                if force_next_highlight:
+                    end_cursor.setPosition(next_block.position() + 1)
+                else:
+                    self.requests.popleft()
+                return
         finally:
-            doc.contentsChange.connect(self.reformat_blocks)
+            self.ignore_requests = False
+            QTimer.singleShot(0, self.do_one_block)
+
+    def join(self):
+        ''' Blocks until all pending highlighting requests are handled '''
+        doc = self.doc
+        if doc is None:
+            self.requests.clear()
+            return
+        self.ignore_requests = True
+        try:
+            while self.requests:
+                start_cursor, end_cursor = self.requests.popleft()
+                block = start_cursor.block()
+                last_block = end_cursor.block()
+                if not last_block.isValid():
+                    last_block = doc.lastBlock()
+                end_pos = last_block.position() + last_block.length()
+                force_next_highlight = False
+                while block.isValid() and (force_next_highlight or block.position() < end_pos):
+                    formats, force_next_highlight = self.parse_single_block(block)
+                    self.apply_format_changes(block, formats)
+                    doc.markContentsDirty(block.position(), block.length())
+                    block = block.next()
+        finally:
+            self.ignore_requests = False
 
     def parse_single_block(self, block):
         ud, is_new_ud = self.get_user_data(block)

src/calibre/gui2/tweak_book/editor/syntax/html.py

@@ -554,9 +554,11 @@ def profile():
     theme = get_theme(tprefs['editor_theme'])
     h.apply_theme(theme)
     h.set_document(doc)
+    h.join()
     import cProfile
     print ('Running profile on', sys.argv[-2])
-    cProfile.runctx('h.rehighlight()', {}, {'h':h}, sys.argv[-1])
+    h.rehighlight()
+    cProfile.runctx('h.join()', {}, {'h':h}, sys.argv[-1])
     print ('Stats saved to:', sys.argv[-1])
     del h
     del doc

src/calibre/gui2/tweak_book/editor/text.py

@@ -265,7 +265,7 @@ def update_extra_selections(self, instant=True):
             sel.append(self.current_cursor_line)
         if self.current_search_mark is not None:
             sel.append(self.current_search_mark)
-        if instant:
+        if instant and not self.highlighter.has_requests:
             sel.extend(self.smarts.get_extra_selections(self))
         else:
             self.smarts_highlight_timer.start()