0.8.63+

recipes/fhm_uk.recipe

@@ -18,15 +18,15 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
     keep_only_tags = [
                dict(name='h1'),
                dict(name='img',attrs={'id' : 'ctl00_Body_imgMainImage'}),
-               dict(name='div',attrs={'id' : ['articleLeft']}),
-                               dict(name='div',attrs={'class' : ['imagesCenterArticle','containerCenterArticle','articleBody']}),
+               dict(name='div',attrs={'id' : ['profileLeft','articleLeft','profileRight','profileBody']}),
+               dict(name='div',attrs={'class' : ['imagesCenterArticle','containerCenterArticle','articleBody',]}),
 
         ]
 
-    #remove_tags    = [
-                              #dict(attrs={'class' : ['player']}),
+    remove_tags    = [
+                              dict(attrs={'id' : ['ctl00_Body_divSlideShow' ]}),
 
-    #]
+    ]
     feeds          = [
         (u'Homepage 1',u'http://feed43.com/6655867614547036.xml'),
         (u'Homepage 2',u'http://feed43.com/4167731873103110.xml'),
@@ -34,7 +34,7 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
         (u'Homepage 4',u'http://feed43.com/6550421522527341.xml'),
         (u'Funny - The Very Best Of The Internet',u'http://feed43.com/4538510106331565.xml'),
         (u'Gaming',u'http://feed43.com/6537162612465672.xml'),
-        (u'Girls',u'http://feed43.com/3674777224513254.xml'),
+        (u'Girls',u'http://feed43.com/4574262733341068.xml'),# edit link http://feed43.com/feed.html?name=4574262733341068
                            ]
 
     extra_css = '''

src/calibre/devices/eb600/driver.py

@@ -101,7 +101,7 @@ class POCKETBOOK360(EB600):
 
     VENDOR_NAME = ['PHILIPS', '__POCKET', 'POCKETBO']
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['MASS_STORGE', 'BOOK_USB_STORAGE',
-            'OK_POCKET_611_61']
+            'OK_POCKET_611_61', 'OK_POCKET_360+61']
 
     OSX_MAIN_MEM = OSX_CARD_A_MEM = 'Philips Mass Storge Media'
     OSX_MAIN_MEM_VOL_PAT = re.compile(r'/Pocket')

src/calibre/devices/errors.py

@@ -48,6 +48,19 @@ def custom_dialog(self, parent):
         '''
         raise NotImplementedError
 
+class InitialConnectionError(OpenFeedback):
+    """ Errors detected during connection after detection but before open, for
+    e.g. in the is_connected() method. """
+
+class OpenFailed(ProtocolError):
+    """ Raised when device cannot be opened this time. No retry is to be done.
+        The device should continue to be polled for future opens. If the
+        message is empty, no exception trace is produced. """
+
+    def __init__(self, msg):
+        ProtocolError.__init__(self, msg)
+        self.show_me = bool(msg and msg.strip())
+
 class DeviceBusy(ProtocolError):
     """ Raised when device is busy """
     def __init__(self, uerr=""):

src/calibre/devices/smart_device_app/driver.py

@@ -14,7 +14,8 @@
 
 from calibre import prints
 from calibre.constants import numeric_version, DEBUG
-from calibre.devices.errors import OpenFeedback
+from calibre.devices.errors import (OpenFailed, ControlError, TimeoutError,
+                                    InitialConnectionError)
 from calibre.devices.interface import DevicePlugin
 from calibre.devices.usbms.books import Book, BookList
 from calibre.devices.usbms.deviceconfig import DeviceConfig
@@ -82,6 +83,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     BASE_PACKET_LEN             = 4096
     PROTOCOL_VERSION            = 1
     MAX_CLIENT_COMM_TIMEOUT     = 60.0 # Wait at most N seconds for an answer
+    MAX_UNSUCCESSFUL_CONNECTS   = 5
 
     opcodes = {
         'NOOP'                   : 12,
@@ -353,24 +355,18 @@ def _call_client(self, op, arg, print_debug_info=True):
             self._debug('protocol error -- empty json string')
         except socket.timeout:
             self._debug('timeout communicating with device')
-            self.device_socket.close()
-            self.device_socket = None
-            self.is_connected = False
-            raise IOError(_('Device did not respond in reasonable time'))
+            self._close_device_socket()
+            raise TimeoutError('Device did not respond in reasonable time')
         except socket.error:
             self._debug('device went away')
-            self.device_socket.close()
-            self.device_socket = None
-            self.is_connected = False
-            raise IOError(_('Device closed the network connection'))
+            self._close_device_socket()
+            raise ControlError('Device closed the network connection')
         except:
             self._debug('other exception')
             traceback.print_exc()
-            self.device_socket.close()
-            self.device_socket = None
-            self.is_connected = False
+            self._close_device_socket()
             raise
-        raise IOError('Device responded with incorrect information')
+        raise ControlError('Device responded with incorrect information')
 
     # Write a file as a series of base64-encoded strings.
     def _put_file(self, infile, lpath, book_metadata, this_book, total_books):
@@ -449,8 +445,16 @@ def _set_known_metadata(self, book, remove=False):
         else:
             self.known_metadata[lpath] = book.deepcopy()
 
-    # The public interface methods.
+    def _close_device_socket(self):
+        if self.device_socket is not None:
+            try:
+                self.device_socket.close()
+            except:
+                pass
+            self.device_socket = None
+        self.is_connected = False
 
+    # The public interface methods.
 
     @synchronous('sync_lock')
     def is_usb_connected(self, devices_on_system, debug=False, only_presence=False):
@@ -471,11 +475,9 @@ def is_usb_connected(self, devices_on_system, debug=False, only_presence=False):
                     pass
             try:
                 if self._call_client('NOOP', dict())[0] is None:
-                    self.is_connected = False
+                    self._close_device_socket()
             except:
-                self.is_connected = False
-            if not self.is_connected:
-                self.device_socket.close()
+                self._close_device_socket()
             return (self.is_connected, self)
         if getattr(self, 'listen_socket', None) is not None:
             ans = select.select((self.listen_socket,), (), (), 0)
@@ -490,23 +492,35 @@ def is_usb_connected(self, devices_on_system, debug=False, only_presence=False):
                     self.listen_socket.settimeout(None)
                     self.device_socket.settimeout(None)
                     self.is_connected = True
+                    try:
+                        peer = self.device_socket.getpeername()[0]
+                        attempts = self.connection_attempts.get(peer, 0)
+                        if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
+                            self._debug('too many connection attempts from', peer)
+                            self._close_device_socket()
+                            raise InitialConnectionError(_('Too many connection attempts from %s')%peer)
+                        else:
+                            self.connection_attempts[peer] = attempts + 1
+                    except InitialConnectionError:
+                        raise
+                    except:
+                        pass
                 except socket.timeout:
-                    if self.device_socket is not None:
-                        self.device_socket.close()
-                        self.is_connected = False
+                    self._close_device_socket()
                 except socket.error:
                     x = sys.exc_info()[1]
                     self._debug('unexpected socket exception', x.args[0])
-                    if self.device_socket is not None:
-                        self.device_socket.close()
-                        self.is_connected = False
+                    self._close_device_socket()
                     raise
-                return (True, self)
+                return (self.is_connected, self)
         return (False, None)
 
     @synchronous('sync_lock')
     def open(self, connected_device, library_uuid):
         self._debug()
+        if not self.is_connected:
+            # We have been called to retry the connection. Give up immediately
+            raise ControlError('Attempt to open a closed device')
         self.current_library_uuid = library_uuid
         self.current_library_name = current_library_name()
         try:
@@ -530,28 +544,24 @@ def open(self, connected_device, library_uuid):
                 # Something wrong with the return. Close the socket
                 # and continue.
                 self._debug('Protocol error - Opcode not OK')
-                self.device_socket.close()
-                self.is_connected = False
+                self._close_device_socket()
                 return False
             if not result.get('versionOK', False):
                 # protocol mismatch
                 self._debug('Protocol error - protocol version mismatch')
-                self.device_socket.close()
-                self.is_connected = False
+                self._close_device_socket()
                 return False
             if result.get('maxBookContentPacketLen', 0) <= 0:
                 # protocol mismatch
                 self._debug('Protocol error - bogus book packet length')
-                self.device_socket.close()
-                self.is_connected = False
+                self._close_device_socket()
                 return False
             self.max_book_packet_len = result.get('maxBookContentPacketLen',
                                                   self.BASE_PACKET_LEN)
             exts = result.get('acceptedExtensions', None)
             if exts is None or not isinstance(exts, list) or len(exts) == 0:
                 self._debug('Protocol error - bogus accepted extensions')
-                self.device_socket.close()
-                self.is_connected = False
+                self._close_device_socket()
                 return False
             self.FORMATS = exts
             if password:
@@ -559,25 +569,34 @@ def open(self, connected_device, library_uuid):
                 if result.get('passwordHash', None) is None:
                     # protocol mismatch
                     self._debug('Protocol error - missing password hash')
-                    self.device_socket.close()
-                    self.is_connected = False
+                    self._close_device_socket()
                     return False
                 if returned_hash != hash_digest:
                     # bad password
                     self._debug('password mismatch')
-                    self._call_client("DISPLAY_MESSAGE", {'messageKind':1})
-                    self.is_connected = False
-                    self.device_socket.close()
-                    raise OpenFeedback('Incorrect password supplied')
+                    try:
+                        self._call_client("DISPLAY_MESSAGE",
+                                {'messageKind':1,
+                                 'currentLibraryName': self.current_library_name,
+                                 'currentLibraryUUID': library_uuid})
+                    except:
+                        pass
+                    self._close_device_socket()
+                    # Don't bother with a message. The user will be informed on
+                    # the device.
+                    raise OpenFailed('')
+            try:
+                peer = self.device_socket.getpeername()
+                self.connection_attempts[peer] = 0
+            except:
+                pass
             return True
         except socket.timeout:
-            self.device_socket.close()
-            self.is_connected = False
+            self._close_device_socket()
         except socket.error:
             x = sys.exc_info()[1]
             self._debug('unexpected socket exception', x.args[0])
-            self.device_socket.close()
-            self.is_connected = False
+            self._close_device_socket()
             raise
         return False
 
@@ -656,7 +675,7 @@ def books(self, oncard=None, end_session=True):
                     self._set_known_metadata(book)
                     bl.add_book(book, replace_metadata=True)
                 else:
-                    raise IOError(_('Protocol error -- book metadata not returned'))
+                    raise ControlError('book metadata not returned')
         return bl
 
     @synchronous('sync_lock')
@@ -675,15 +694,12 @@ def sync_booklists(self, booklists, end_session=True):
                                                   print_debug_info=False)
                 if opcode != 'OK':
                     self._debug('protocol error', opcode, i)
-                    raise IOError(_('Protocol error -- sync_booklists'))
+                    raise ControlError('sync_booklists')
 
     @synchronous('sync_lock')
     def eject(self):
         self._debug()
-        if self.device_socket:
-            self.device_socket.close()
-            self.device_socket = None
-            self.is_connected = False
+        self._close_device_socket()
 
     @synchronous('sync_lock')
     def post_yank_cleanup(self):
@@ -706,7 +722,7 @@ def upload_books(self, files, names, on_card=None, end_session=True,
             book = Book(self.PREFIX, lpath, other=mdata)
             length = self._put_file(infile, lpath, book, i, len(files))
             if length < 0:
-                raise IOError(_('Sending book %s to device failed') % lpath)
+                raise ControlError('Sending book %s to device failed' % lpath)
             paths.append((lpath, length))
             # No need to deal with covers. The client will get the thumbnails
             # in the mi structure
@@ -747,7 +763,7 @@ def delete_books(self, paths, end_session=True):
             if opcode == 'OK':
                 self._debug('removed book with UUID', result['uuid'])
             else:
-                raise IOError(_('Protocol error - delete books'))
+                raise ControlError('Protocol error - delete books')
 
     @synchronous('sync_lock')
     def remove_books_from_metadata(self, paths, booklists):
@@ -783,7 +799,7 @@ def get_file(self, path, outfile, end_session=True):
                 else:
                     eof = True
             else:
-                raise IOError(_('request for book data failed'))
+                raise ControlError('request for book data failed')
 
     @synchronous('sync_lock')
     def set_plugboards(self, plugboards, pb_func):
@@ -811,6 +827,7 @@ def startup_on_demand(self):
         self.debug_start_time = time.time()
         self.max_book_packet_len = 0
         self.noop_counter = 0
+        self.connection_attempts = {}
         try:
             self.listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         except:

src/calibre/ebooks/mobi/writer2/serializer.py

@@ -7,11 +7,10 @@
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re
+import re, unicodedata
 
 from calibre.ebooks.oeb.base import (OEB_DOCS, XHTML, XHTML_NS, XML_NS,
         namespace, prefixname, urlnormalize)
-from calibre.ebooks import normalize
 from calibre.ebooks.mobi.mobiml import MBP_NS
 from calibre.ebooks.mobi.utils import is_guide_ref_start
 
@@ -356,7 +355,9 @@ def serialize_text(self, text, quot=False):
         text = text.replace(u'\u00AD', '') # Soft-hyphen
         if quot:
             text = text.replace('"', '&quot;')
-        self.buf.write(normalize(text).encode('utf-8'))
+        if isinstance(text, unicode):
+            text = unicodedata.normalize('NFC', text)
+        self.buf.write(text.encode('utf-8'))
 
     def fixup_links(self):
         '''

src/calibre/gui2/device.py

@@ -13,7 +13,8 @@
 from calibre.customize.ui import (available_input_formats, available_output_formats,
     device_plugins)
 from calibre.devices.interface import DevicePlugin
-from calibre.devices.errors import UserFeedback, OpenFeedback
+from calibre.devices.errors import (UserFeedback, OpenFeedback, OpenFailed,
+                                    InitialConnectionError)
 from calibre.gui2.dialogs.choose_format_device import ChooseFormatDeviceDialog
 from calibre.utils.ipc.job import BaseJob
 from calibre.devices.scanner import DeviceScanner
@@ -172,6 +173,8 @@ def do_connect(self, connected_devices, device_kind):
                     self.open_feedback_msg(dev.get_gui_name(), e)
                     self.ejected_devices.add(dev)
                 continue
+            except OpenFailed:
+                raise
             except:
                 tb = traceback.format_exc()
                 if DEBUG or tb not in self.reported_errors:
@@ -225,24 +228,32 @@ def detect_device(self):
                             only_presence=True, debug=True)
                 self.connected_device_removed()
         else:
-            possibly_connected_devices = []
-            for device in self.devices:
-                if device in self.ejected_devices:
-                    continue
-                possibly_connected, detected_device = \
-                        self.scanner.is_device_connected(device)
-                if possibly_connected:
-                    possibly_connected_devices.append((device, detected_device))
-            if possibly_connected_devices:
-                if not self.do_connect(possibly_connected_devices,
-                                       device_kind='device'):
-                    if DEBUG:
-                        prints('Connect to device failed, retrying in 5 seconds...')
-                    time.sleep(5)
+            try:
+                possibly_connected_devices = []
+                for device in self.devices:
+                    if device in self.ejected_devices:
+                        continue
+                    try:
+                        possibly_connected, detected_device = \
+                                self.scanner.is_device_connected(device)
+                    except InitialConnectionError as e:
+                        self.open_feedback_msg(device.get_gui_name(), e)
+                        continue
+                    if possibly_connected:
+                        possibly_connected_devices.append((device, detected_device))
+                if possibly_connected_devices:
                     if not self.do_connect(possibly_connected_devices,
-                                       device_kind='usb'):
+                                           device_kind='device'):
                         if DEBUG:
-                            prints('Device connect failed again, giving up')
+                            prints('Connect to device failed, retrying in 5 seconds...')
+                        time.sleep(5)
+                        if not self.do_connect(possibly_connected_devices,
+                                           device_kind='usb'):
+                            if DEBUG:
+                                prints('Device connect failed again, giving up')
+            except OpenFailed as e:
+                if e.show_me:
+                    traceback.print_exc()
 
     # Mount devices that don't use USB, such as the folder device and iTunes
     # This will be called on the GUI thread. Because of this, we must store

src/calibre/library/caches.py

@@ -568,6 +568,7 @@ def get_keypair_matches(self, location, query, candidates):
                     matches.add(id_)
                 continue
 
+            add_if_nothing_matches = valq == 'false'
             pairs = [p.strip() for p in item[loc].split(split_char)]
             for pair in pairs:
                 parts = pair.split(':')
@@ -583,10 +584,14 @@ def get_keypair_matches(self, location, query, candidates):
                             continue
                     elif valq == 'false':
                         if v:
+                            add_if_nothing_matches = False
                             continue
                     elif not _match(valq, v, valq_mkind):
                         continue
                 matches.add(id_)
+
+            if add_if_nothing_matches:
+                matches.add(id_)
         return matches
 
     def _matchkind(self, query):

src/calibre/library/server/opds.py

@@ -561,7 +561,9 @@ def opds_category(self, category=None, which=None, version=0, offset=0):
         if type_ != 'I':
             raise cherrypy.HTTPError(404, 'Non id categories not supported')
 
-        ids = self.db.get_books_for_category(category, which)
+        q = category
+        if q == 'news': q = 'tags'
+        ids = self.db.get_books_for_category(q, which)
         sort_by = 'series' if category == 'series' else 'title'
 
         return self.get_opds_acquisition_feed(ids, offset, page_url,