Pull from Trunk

resources/recipes/china_press.recipe

@@ -0,0 +1,71 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1277228948(BasicNewsRecipe):
+    title          = u'China Press USA'
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    __author__            = 'rty'
+    __version__            = '1.0'
+    language = 'zh_CN'
+    pubisher  = 'www.chinapressusa.com'
+    description           = 'Overseas Chinese Network Newspaper in the USA'
+    category              = 'News in Chinese, USA'
+    remove_javascript = True
+    use_embedded_content   = False
+    no_stylesheets = True
+    #encoding               = 'GB2312'
+    encoding               = 'UTF-8'
+    conversion_options = {'linearize_tables':True}
+    masthead_url ='http://www.chinapressusa.com/common/images/logo.gif'
+    extra_css = '''
+             @font-face { font-family: "DroidFont", serif, sans-serif;  src: url(res:///system/fonts/DroidSansFallback.ttf); }\n
+             body {
+                  margin-right: 8pt;
+                  font-family: 'DroidFont', serif;}
+              h1  {font-family: 'DroidFont', serif, sans-serif}
+            .show {font-family: 'DroidFont', serif, sans-serif}
+            '''
+    feeds          = [
+	(u'\u65b0\u95fb\u9891\u9053', u'http://news.uschinapress.com/news.xml'),
+	(u'\u534e\u4eba\u9891\u9053', u'http://chinese.uschinapress.com/chinese.xml'),
+	(u'\u8bc4\u8bba\u9891\u9053', u'http://review.uschinapress.com/review.xml'),
+	]
+    keep_only_tags = [
+                              dict(name='div', attrs={'class':'show'}),
+                               ]
+    remove_tags = [
+     #               dict(name='table', attrs={'class':'xle'}),
+                    dict(name='div', attrs={'class':'time'}),
+                         ]
+    remove_tags_after = [
+                  dict(name='div', attrs={'class':'bank17'}),
+         #         dict(name='a', attrs={'class':'ab12'}),
+                         ]
+
+
+    def append_page(self, soup, appendtag, position):
+        pager = soup.find('div',attrs={'id':'displaypagenum'})
+        if pager:
+           nexturl = self.INDEX + pager.a['href']
+           soup2 = self.index_to_soup(nexturl)
+           texttag = soup2.find('div', attrs={'class':'show'})
+           for it in texttag.findAll(style=True):
+               del it['style']
+           newpos = len(texttag.contents)
+           self.append_page(soup2,texttag,newpos)
+           texttag.extract()
+           appendtag.insert(position,texttag)
+
+
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="Content-Language" content="zh-CN"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
+        soup.head.insert(0,mtag)
+
+        for item in soup.findAll(style=True):
+            del item['style']
+        self.append_page(soup, soup.body, 3)
+        pager = soup.find('div',attrs={'id':'displaypagenum'})
+        if pager:
+           pager.extract()
+        return soup

resources/recipes/london_free_press.recipe

@@ -0,0 +1,38 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LondonFreePress(BasicNewsRecipe):
+    title          = u'London Free Press'
+    __author__            = 'rty'
+    oldest_article = 4
+    max_articles_per_feed = 100
+
+    pubisher  = 'lfpress.com'
+    description           = 'Ontario Canada Newspaper'
+    category              = 'News, Ontario, Canada'
+    remove_javascript = True
+    use_embedded_content   = False
+    no_stylesheets = True
+    language = 'en_CA'
+    encoding               = 'utf-8'
+    conversion_options = {'linearize_tables':True}
+
+    feeds          = [
+                    (u'News', u'http://www.lfpress.com/news/rss.xml'),
+                    (u'Comment', u'http://www.lfpress.com/comment/rss.xml'),
+                    (u'Entertainment', u'http://www.lfpress.com/entertainment/rss.xml '),
+                    (u'Money', u'http://www.lfpress.com/money/rss.xml '),
+                    (u'Life', u'http://www.lfpress.com/life/rss.xml '),
+                    (u'Sports', u'http://www.lfpress.com/sports/rss.xml ')
+                        ]
+
+    keep_only_tags = [
+                              dict(name='div', attrs={'id':'article'}),
+                               ]
+    remove_tags = [
+                    dict(name='div', attrs={'id':'commentsBottom'}),
+                    dict(name='div', attrs={'class':['leftBox','bottomBox clear']}),
+   	dict(name='ul', attrs={'class':'tabs dl contentSwap'}),
+                         ]
+    remove_tags_after = [
+                  dict(name='div', attrs={'class':'bottomBox clear'}),
+                         ]

resources/recipes/losservatoreromano_it.recipe

@@ -0,0 +1,48 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.vatican.va/news_services/or/or_quo
+'''
+
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LOsservatoreRomano_it(BasicNewsRecipe):
+    title                = "L'Osservatore Romano"
+    __author__           = 'Darko Miletic'
+    description          = 'Quiornale quotidiano, politico, religioso del Vaticano'
+    publisher            = 'La Santa Sede'
+    category             = 'news, politics, religion, Vatican'
+    no_stylesheets       = True
+    INDEX                = 'http://www.vatican.va'
+    FEEDPAGE             = INDEX + '/news_services/or/or_quo/index.html'
+    CONTENTPAGE          = INDEX + '/news_services/or/or_quo/text.html'
+    use_embedded_content = False
+    encoding             = 'cp1252'
+    language             = 'it'
+    publication_type     = 'newspaper'
+
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        , 'linearize_tables' : True
+                        }
+
+    def parse_index(self):
+        articles = []
+        articles.append({
+                          'title'      :self.title
+                         ,'date'       :''
+                         ,'url'        :self.CONTENTPAGE
+                         ,'description':''
+                        })
+        return [(self.title, articles)]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return self.adeify_images(soup)
+

resources/recipes/lrb.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
 lrb.co.uk
 '''
@@ -8,32 +8,38 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class LondonReviewOfBooks(BasicNewsRecipe):
-    title                 = u'London Review of Books'
-    __author__            = u'Darko Miletic'
-    description           = u'Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
-    category              = 'news, literature, England'
-    publisher             = 'London Review of Books'
-    oldest_article        = 7
+    title                 = 'London Review of Books (free)'
+    __author__            = 'Darko Miletic'
+    description           = 'Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
+    category              = 'news, literature, UK'
+    publisher             = 'LRB ltd.'
+    oldest_article        = 15
     max_articles_per_feed = 100
     language              = 'en_GB'
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
+    publication_type      = 'magazine'
+    masthead_url          = 'http://www.lrb.co.uk/assets/images/lrb_logo_big.gif'
+    extra_css             = ' body{font-family: Georgia,Palatino,"Palatino Linotype",serif} '
 
-    conversion_options = {  
+    conversion_options = {
                              'comments'  : description
                             ,'tags'      : category
                             ,'language'  : language
                             ,'publisher' : publisher
                          }
-    
-    keep_only_tags = [dict(name='div' , attrs={'id'   :'main'})]
-    remove_tags = [
-                    dict(name='div' , attrs={'class':['pagetools','issue-nav-controls','nocss']})
-                   ,dict(name='div' , attrs={'id'   :['mainmenu','precontent','otherarticles']     })
-                   ,dict(name='span', attrs={'class':['inlineright','article-icons']})
-                   ,dict(name='ul'  , attrs={'class':'article-controls'})
-                   ,dict(name='p'   , attrs={'class':'meta-info'       })
-                  ]
+
+    keep_only_tags = [dict(attrs={'class':['article-body indent','letters','article-list']})]
+    remove_attributes = ['width','height']
 
     feeds = [(u'London Review of Books', u'http://www.lrb.co.uk/lrbrss.xml')]
+
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup('http://www.lrb.co.uk/')
+        cover_item = soup.find('p',attrs={'class':'cover'})
+        if cover_item:
+           cover_url = 'http://www.lrb.co.uk' + cover_item.a.img['src']
+        return cover_url
+

resources/recipes/lrb_payed.recipe

@@ -0,0 +1,75 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+lrb.co.uk
+'''
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LondonReviewOfBooksPayed(BasicNewsRecipe):
+    title                 = 'London Review of Books'
+    __author__            = 'Darko Miletic'
+    description           = 'Subscription content. Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
+    category              = 'news, literature, UK'
+    publisher             = 'LRB Ltd.'
+    max_articles_per_feed = 100
+    language              = 'en_GB'
+    no_stylesheets        = True
+    delay                 = 1
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    INDEX                 = 'http://www.lrb.co.uk'
+    LOGIN                 = INDEX + '/login'
+    masthead_url          = INDEX + '/assets/images/lrb_logo_big.gif'
+    needs_subscription    = True
+    publication_type      = 'magazine'
+    extra_css             = ' body{font-family: Georgia,Palatino,"Palatino Linotype",serif} '
+
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open(self.LOGIN)
+            br.select_form(nr=1)
+            br['username'] = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+
+    def parse_index(self):
+        articles = []
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('p',attrs={'class':'cover'})
+        lrbtitle = self.title
+        if  cover_item:
+            self.cover_url = self.INDEX + cover_item.a.img['src']
+            content = self.INDEX + cover_item.a['href']
+            soup2 = self.index_to_soup(content)
+            sitem = soup2.find(attrs={'class':'article-list'})
+            lrbtitle = soup2.head.title.string
+            for item in sitem.findAll('a',attrs={'class':'title'}):
+                description = u''
+                title_prefix = u''
+                feed_link = item
+                if feed_link.has_key('href'):
+                    url   = self.INDEX + feed_link['href']
+                    title = title_prefix + self.tag_to_string(feed_link)
+                    date  = strftime(self.timefmt)
+                    articles.append({
+                                      'title'      :title
+                                     ,'date'       :date
+                                     ,'url'        :url
+                                     ,'description':description
+                                    })
+        return [(lrbtitle, articles)]
+
+    conversion_options = {
+                             'comments'  : description
+                            ,'tags'      : category
+                            ,'language'  : language
+                            ,'publisher' : publisher
+                         }
+
+    keep_only_tags = [dict(name='div' , attrs={'class':['article-body indent','letters']})]
+    remove_attributes = ['width','height']

resources/recipes/national_post.recipe

@@ -7,18 +7,18 @@ class NYTimes(BasicNewsRecipe):
     __author__  = 'Krittika Goyal'
     description = 'Canadian national newspaper'
     timefmt = ' [%d %b, %Y]'
-    needs_subscription = False
     language = 'en_CA'
+    needs_subscription = False
 
     no_stylesheets = True
     #remove_tags_before = dict(name='h1', attrs={'class':'heading'})
-    #remove_tags_after  = dict(name='td', attrs={'class':'newptool1'})
+    remove_tags_after  = dict(name='div', attrs={'class':'npStoryTools npWidth1-6 npRight npTxtStrong'})
     remove_tags = [
        dict(name='iframe'),
-       dict(name='div', attrs={'class':'story-tools'}),
+       dict(name='div', attrs={'class':['story-tools', 'npStoryTools npWidth1-6 npRight npTxtStrong']}),
        #dict(name='div', attrs={'id':['qrformdiv', 'inSection', 'alpha-inner']}),
        #dict(name='form', attrs={'onsubmit':''}),
-       #dict(name='table', attrs={'cellspacing':'0'}),
+       dict(name='ul', attrs={'class':'npTxtAlt npGroup npTxtCentre npStoryShare npTxtStrong npTxtDim'}),
     ]
 
    # def preprocess_html(self, soup):
@@ -37,7 +37,7 @@ def nejm_get_index(self):
     def parse_index(self):
             soup = self.nejm_get_index()
 
-            div = soup.find(id='LegoText4')
+            div = soup.find(id='npContentMain')
 
             current_section = None
             current_articles = []
@@ -50,7 +50,7 @@ def parse_index(self):
                     current_section = self.tag_to_string(x)
                     current_articles = []
                     self.log('\tFound section:', current_section)
-                if current_section is not None and x.name == 'h3':
+                if current_section is not None and x.name == 'h5':
                     # Article found
                     title = self.tag_to_string(x)
                     a = x.find('a', href=lambda x: x and 'story' in x)
@@ -59,8 +59,8 @@ def parse_index(self):
                     url = a.get('href', False)
                     if not url or not title:
                         continue
-                    if url.startswith('story'):
-                         url = 'http://www.nationalpost.com/todays-paper/'+url
+                    #if url.startswith('story'):
+                    url = 'http://www.nationalpost.com/todays-paper/'+url
                     self.log('\t\tFound article:', title)
                     self.log('\t\t\t', url)
                     current_articles.append({'title': title, 'url':url,
@@ -70,28 +70,11 @@ def parse_index(self):
                 feeds.append((current_section, current_articles))
 
             return feeds
-
     def preprocess_html(self, soup):
-        story = soup.find(name='div', attrs={'class':'triline'})
-        page2_link = soup.find('p','pagenav')
-        if page2_link:
-            atag = page2_link.find('a',href=True)
-            if atag:
-                page2_url = atag['href']
-                if page2_url.startswith('story'):
-                         page2_url = 'http://www.nationalpost.com/todays-paper/'+page2_url
-                elif page2_url.startswith( '/todays-paper/story.html'):
-                    page2_url = 'http://www.nationalpost.com/'+page2_url
-                page2_soup = self.index_to_soup(page2_url)
-                if page2_soup:
-                    page2_content = page2_soup.find('div','story-content')
-                    if page2_content:
-                        full_story = BeautifulSoup('<div></div>')
-                        full_story.insert(0,story)
-                        full_story.insert(1,page2_content)
-                        story = full_story
+        story = soup.find(name='div', attrs={'id':'npContentMain'})
+        ##td = heading.findParent(name='td')
+        ##td.extract()
         soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
         body = soup.find(name='body')
         body.insert(0, story)
         return soup
-

resources/recipes/new_scientist.recipe

@@ -32,15 +32,16 @@ class NewScientist(BasicNewsRecipe):
                         }
     preprocess_regexps = [(re.compile(r'</title>.*?</head>', re.DOTALL|re.IGNORECASE),lambda match: '</title></head>')]
 
-    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','nsblgposts','hldgalcols']})]
+    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})]
 
     remove_tags = [
                      dict(name='div'  , attrs={'class':['hldBd','adline','pnl','infotext' ]})
-                    ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools']})
+                    ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools','comments','blgsocial']})
                     ,dict(name='p'    , attrs={'class':['marker','infotext'               ]})
                     ,dict(name='meta' , attrs={'name' :'description'                       })
+                    ,dict(name='a'    , attrs={'rel'  :'tag'                                })
                   ]
-    remove_tags_after = dict(attrs={'class':'nbpcopy'})
+    remove_tags_after = dict(attrs={'class':['nbpcopy','comments']})
     remove_attributes = ['height','width']
 
     feeds          = [

resources/recipes/nytimes.recipe

@@ -17,7 +17,7 @@ class NYTimes(BasicNewsRecipe):
     title       = 'New York Times Top Stories'
     __author__  = 'GRiker'
     language = 'en'
-    requires_version = (0, 7, 3)
+    requires_version = (0, 7, 5)
     description = 'Top Stories from the New York Times'
 
     # List of sections typically included in Top Stories.  Use a keyword from the
@@ -79,6 +79,7 @@ class NYTimes(BasicNewsRecipe):
                             'doubleRule',
                             'dottedLine',
                             'entry-meta',
+                            'entry-response module',
                             'icon enlargeThis',
                             'leftNavTabs',
                             'module box nav',
@@ -110,6 +111,7 @@ class NYTimes(BasicNewsRecipe):
                             'navigation',
                             'portfolioInline',
                             'relatedArticles',
+                            'respond',
                             'side_search',
                             'side_index',
                             'side_tool',

resources/recipes/nytimes_sub.recipe

@@ -13,14 +13,14 @@
 import re, string, time
 from calibre import strftime
 from calibre.web.feeds.recipes import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, NavigableString, Tag
+from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup, NavigableString, Tag
 
 class NYTimes(BasicNewsRecipe):
 
     title       = 'The New York Times'
     __author__  = 'GRiker'
     language = 'en'
-    requires_version = (0, 7, 3)
+    requires_version = (0, 7, 5)
 
     description = 'Daily news from the New York Times (subscription version)'
     allSectionKeywords = ['The Front Page', 'International','National','Obituaries','Editorials',
@@ -66,6 +66,7 @@ class NYTimes(BasicNewsRecipe):
                             'doubleRule',
                             'dottedLine',
                             'entry-meta',
+                            'entry-response module',
                             'icon enlargeThis',
                             'leftNavTabs',
                             'module box nav',
@@ -97,6 +98,7 @@ class NYTimes(BasicNewsRecipe):
                             'navigation',
                             'portfolioInline',
                             'relatedArticles',
+                            'respond',
                             'side_search',
                             'side_index',
                             'side_tool',
@@ -417,12 +419,11 @@ def postprocess_html(self,soup, True):
 
         return soup
 
-    def postprocess_book(self, oeb, opts, log) :
-        print "\npostprocess_book()\n"
-
-        def extract_byline(href) :
-            # <meta name="byline" content=
-            soup = BeautifulSoup(str(oeb.manifest.hrefs[href]))
+    def populate_article_metadata(self,article,soup,first):
+        '''
+        Extract author and description from article, add to article metadata
+        '''
+        def extract_author(soup):
             byline = soup.find('meta',attrs={'name':['byl','CLMST']})
             if byline :
                 author = byline['content']
@@ -432,50 +433,32 @@ def extract_byline(href) :
                 if byline:
                     author = byline.renderContents()
                 else:
-                    print "couldn't find byline in %s" % href
                     print soup.prettify()
                     return None
-            # Kill commas - Kindle switches to '&'
-            return re.sub(',','',author)
+            return author
 
-        def extract_description(href) :
-            soup = BeautifulSoup(str(oeb.manifest.hrefs[href]))
+        def extract_description(soup):
             description = soup.find('meta',attrs={'name':['description','description ']})
             if description :
-#                print repr(description['content'])
-#                print self.massageNCXText(description['content'])
                 return self.massageNCXText(description['content'])
             else:
                 # Take first paragraph of article
-                articleBody = soup.find('div',attrs={'id':'articleBody'})
-                if not articleBody:
+                articlebody = soup.find('div',attrs={'id':'articlebody'})
+                if not articlebody:
                     # Try again with class instead of id
-                    articleBody = soup.find('div',attrs={'class':'articleBody'})
-                    if not articleBody:
-                        print 'postprocess_book.extract_description(): Did not find <div id="articleBody">:'
+                    articlebody = soup.find('div',attrs={'class':'articlebody'})
+                    if not articlebody:
+                        print 'postprocess_book.extract_description(): Did not find <div id="articlebody">:'
                         print soup.prettify()
                         return None
-                paras = articleBody.findAll('p')
+                paras = articlebody.findAll('p')
                 for p in paras:
                     if p.renderContents() > '' :
                         return self.massageNCXText(self.tag_to_string(p,use_alt=False))
                 return None
 
-        # Method entry point here
-        # Single section toc looks different than multi-section tocs
-        if oeb.toc.depth() == 2 :
-            for article in oeb.toc :
-                if article.author is None :
-                    article.author = extract_byline(article.href)
-                if article.description is None :
-                    article.description = extract_description(article.href).decode('utf-8')
-        elif oeb.toc.depth() == 3 :
-            for section in oeb.toc :
-                for article in section :
-                    if article.author is None :
-                        article.author = extract_byline(article.href)
-                    if article.description is None :
-                        article.description = extract_description(article.href)
+        article.author = extract_author(soup)
+        article.summary = article.text_summary = extract_description(soup)
 
     def strip_anchors(self,soup):
         paras = soup.findAll(True)

resources/recipes/people_daily.recipe

@@ -0,0 +1,57 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1277129332(BasicNewsRecipe):
+    title          = u'People Daily - China'
+    oldest_article = 2
+    max_articles_per_feed = 100
+    __author__            = 'rty'
+
+    pubisher  = 'people.com.cn'
+    description           = 'People Daily Newspaper'
+    language = 'zh'
+    category              = 'News, China'
+    remove_javascript = True
+    use_embedded_content   = False
+    no_stylesheets = True
+    encoding               = 'GB2312'
+    conversion_options = {'linearize_tables':True}
+
+    feeds          = [(u'\u56fd\u5185\u65b0\u95fb', u'http://www.people.com.cn/rss/politics.xml'),
+       (u'\u56fd\u9645\u65b0\u95fb', u'http://www.people.com.cn/rss/world.xml'),
+       (u'\u7ecf\u6d4e\u65b0\u95fb', u'http://www.people.com.cn/rss/finance.xml'),
+       (u'\u4f53\u80b2\u65b0\u95fb', u'http://www.people.com.cn/rss/sports.xml'),
+       (u'\u53f0\u6e7e\u65b0\u95fb', u'http://www.people.com.cn/rss/haixia.xml')]
+    keep_only_tags = [
+                              dict(name='div', attrs={'class':'left_content'}),
+                               ]
+    remove_tags = [
+                    dict(name='table', attrs={'class':'title'}),
+                         ]
+    remove_tags_after = [
+                  dict(name='table', attrs={'class':'bianji'}),
+                         ]
+
+    def append_page(self, soup, appendtag, position):
+        pager = soup.find('img',attrs={'src':'/img/next_b.gif'})
+        if pager:
+           nexturl = self.INDEX + pager.a['href']
+           soup2 = self.index_to_soup(nexturl)
+           texttag = soup2.find('div', attrs={'class':'left_content'})
+           #for it in texttag.findAll(style=True):
+           #   del it['style']
+           newpos = len(texttag.contents)
+           self.append_page(soup2,texttag,newpos)
+           texttag.extract()
+           appendtag.insert(position,texttag)
+
+
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="content-type" content="text/html;charset=GB2312" />\n<meta http-equiv="content-language" content="utf-8" />'
+        soup.head.insert(0,mtag)
+        for item in soup.findAll(style=True):
+            del item['form']
+        self.append_page(soup, soup.body, 3)
+        #pager = soup.find('a',attrs={'class':'ab12'})
+        #if pager:
+        #   pager.extract()
+        return soup

src/calibre/customize/profiles.py

@@ -36,7 +36,7 @@ def __init__(self, *args, **kwargs):
         self.fnames = dict((name, sz) for name, _, sz in self.fsizes if name)
         self.fnums = dict((num, sz) for _, num, sz in self.fsizes if num)
 
-
+# Input profiles {{{
 class InputProfile(Plugin):
 
     author = 'Kovid Goyal'
@@ -218,6 +218,8 @@ class NookInput(InputProfile):
 
 input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower()))
 
+# }}}
+
 class OutputProfile(Plugin):
 
     author = 'Kovid Goyal'
@@ -237,11 +239,12 @@ class OutputProfile(Plugin):
     # If True the MOBI renderer on the device supports MOBI indexing
     supports_mobi_indexing = False
 
-    # Device supports displaying a nested TOC
-    supports_nested_toc = True
-
     # If True output should be optimized for a touchscreen interface
     touchscreen = False
+    touchscreen_news_css = ''
+    # A list of extra (beyond CSS 2.1) modules supported by the device
+    # Format is a cssutils profile dictionary (see iPad for example)
+    extra_css_modules = []
 
     @classmethod
     def tags_to_string(cls, tags):
@@ -256,8 +259,94 @@ class iPadOutput(OutputProfile):
     screen_size = (768, 1024)
     comic_screen_size = (768, 1024)
     dpi = 132.0
-    supports_nested_toc = False
+    extra_css_modules = [
+        {
+            'name':'webkit',
+            'props': { '-webkit-border-bottom-left-radius':'{length}',
+                '-webkit-border-bottom-right-radius':'{length}',
+                '-webkit-border-top-left-radius':'{length}',
+                '-webkit-border-top-right-radius':'{length}',
+                '-webkit-border-radius': r'{border-width}(\s+{border-width}){0,3}|inherit',
+            },
+            'macros': {'border-width': '{length}|medium|thick|thin'}
+        }
+    ]
     touchscreen = True
+    # touchscreen_news_css {{{
+    touchscreen_news_css = u'''
+			/* hr used in articles */
+            .caption_divider {
+            	border:#ccc 1px solid;
+				}
+
+            .touchscreen_navbar {
+                background:#ccc;
+                border:#ccc 1px solid;
+                border-collapse:separate;
+                border-spacing:1px;
+                margin-left: 5%;
+                margin-right: 5%;
+                width: 90%;
+                -webkit-border-radius:4px;
+                }
+            .touchscreen_navbar td {
+                background:#fff;
+                font-family:Helvetica;
+                font-size:90%;
+                padding: 5px;
+                text-align:center;
+                }
+            .touchscreen_navbar td:first-child {
+				-webkit-border-top-left-radius:4px;
+				-webkit-border-bottom-left-radius:4px;
+				}
+			.touchscreen_navbar td:last-child {
+				-webkit-border-top-right-radius:4px;
+				-webkit-border-bottom-right-radius:4px;
+				}
+
+            .feed_link {
+                font-style: italic;
+                }
+
+			/* Index formatting */
+			.publish_date {
+				text-align:center;
+				}
+			.divider {
+				border-bottom:1em solid white;
+				border-top:1px solid gray;
+				}
+
+            /* Feed summary formatting */
+            .feed_title {
+                text-align: center;
+                font-size: 160%;
+                }
+
+            .summary_headline {
+                font-weight:bold;
+                text-align:left;
+				}
+
+            .summary_byline {
+                text-align:left;
+                font-family:monospace;
+				}
+
+            .summary_text {
+                text-align:left;
+				}
+
+            .feed {
+                font-family:sans-serif;
+                font-weight:bold;
+                font-size:larger;
+				}
+
+        '''
+        # }}}
+
 
 class SonyReaderOutput(OutputProfile):
 

src/calibre/customize/ui.py

@@ -151,13 +151,13 @@ def reread_filetype_plugins():
 
 
 def _run_filetype_plugins(path_to_file, ft=None, occasion='preprocess'):
-    occasion = {'import':_on_import, 'preprocess':_on_preprocess,
+    occasion_plugins = {'import':_on_import, 'preprocess':_on_preprocess,
                 'postprocess':_on_postprocess}[occasion]
     customization = config['plugin_customization']
     if ft is None:
         ft = os.path.splitext(path_to_file)[-1].lower().replace('.', '')
     nfp = path_to_file
-    for plugin in occasion.get(ft, []):
+    for plugin in occasion_plugins.get(ft, []):
         if is_disabled(plugin):
             continue
         plugin.site_customization = customization.get(plugin.name, '')

src/calibre/devices/android/driver.py

@@ -45,8 +45,8 @@ class ANDROID(USBMS):
             'GT-I5700', 'SAMSUNG']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD',
-            'PROD_GT-I9000']
-    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'PROD_GT-I9000_CARD']
+            'PR OD_GT-I9000']
+    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'PR OD_GT-I9000_CARD']
 
     OSX_MAIN_MEM = 'HTC Android Phone Media'
 

src/calibre/devices/apple/driver.py

@@ -10,13 +10,15 @@
 from calibre import fit_image
 from calibre.constants import isosx, iswindows
 from calibre.devices.errors import UserFeedback
+from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.epub import set_metadata
 from calibre.library.server.utils import strftime
-from calibre.utils.config import Config, config_dir
+from calibre.utils.config import config_dir
 from calibre.utils.date import isoformat, now, parse_date
+from calibre.utils.localization import get_lang
 from calibre.utils.logging import Log
 from calibre.utils.zipfile import ZipFile
 
@@ -33,8 +35,15 @@
 if iswindows:
     import pythoncom, win32com.client
 
+class DriverBase(DeviceConfig, DevicePlugin):
+    # Needed for config_widget to work
+    FORMATS = ['epub', 'pdf']
 
-class ITUNES(DevicePlugin):
+    @classmethod
+    def _config_base_name(cls):
+        return 'iTunes'
+
+class ITUNES(DriverBase):
     '''
     Calling sequences:
     Initialization:
@@ -78,18 +87,22 @@ class ITUNES(DevicePlugin):
     supported_platforms = ['osx','windows']
     author = 'GRiker'
     #: The version of this plugin as a 3-tuple (major, minor, revision)
-    version        = (0,7,0)
+    version        = (0,8,0)
 
     OPEN_FEEDBACK_MESSAGE = _(
         'Apple device detected, launching iTunes, please wait ...')
 
-    FORMATS = ['epub']
 
     # Product IDs:
-    #  0x1292:iPhone 3G
-    #  0x129a:iPad
+    #  0x1291   iPod Touch
+    #  0x1292   iPhone 3G
+    #  0x1293   iPod Touch 2G
+    #  0x1294   iPhone 3GS
+    #  0x1297   iPhone 4
+    #  0x1299   iPod Touch 3G
+    #  0x129a   iPad
     VENDOR_ID = [0x05ac]
-    PRODUCT_ID = [0x129a]
+    PRODUCT_ID = [0x1292,0x1293,0x1294,0x1297,0x1299,0x129a]
     BCD = [0x01]
 
     # iTunes enumerations
@@ -141,6 +154,10 @@ class ITUNES(DevicePlugin):
                 'SongNames',
                 ]
 
+    # Cover art size limits
+    MAX_COVER_WIDTH = 510
+    MAX_COVER_HEIGHT = 680
+
     # Properties
     cached_books = {}
     cache_dir = os.path.join(config_dir, 'caches', 'itunes')
@@ -159,7 +176,6 @@ class ITUNES(DevicePlugin):
     sources = None
     update_msg = None
     update_needed = False
-    use_series_data = True
 
     # Public methods
     def add_books_to_metadata(self, locations, metadata, booklists):
@@ -173,16 +189,17 @@ def add_books_to_metadata(self, locations, metadata, booklists):
                                 (L{books}(oncard=None), L{books}(oncard='carda'),
                                 L{books}(oncard='cardb')).
         '''
+        if DEBUG:
+            self.log.info("ITUNES.add_books_to_metadata()")
 
         task_count = float(len(self.update_list))
 
         # Delete any obsolete copies of the book from the booklist
         if self.update_list:
-            if True:
-                self.log.info("ITUNES.add_books_to_metadata()")
-                #self._dump_booklist(booklists[0], header='before',indent=2)
-                #self._dump_update_list(header='before',indent=2)
-                #self._dump_cached_books(header='before',indent=2)
+            if False:
+                self._dump_booklist(booklists[0], header='before',indent=2)
+                self._dump_update_list(header='before',indent=2)
+                self._dump_cached_books(header='before',indent=2)
 
             for (j,p_book) in enumerate(self.update_list):
                 if False:
@@ -230,12 +247,12 @@ def add_books_to_metadata(self, locations, metadata, booklists):
 
         # Add new books to booklists[0]
         for new_book in locations[0]:
-            if False:
+            if DEBUG:
                 self.log.info("  adding '%s' by '%s' to booklists[0]" %
                     (new_book.title, new_book.author))
             booklists[0].append(new_book)
 
-        if False:
+        if DEBUG:
             self._dump_booklist(booklists[0],header='after',indent=2)
             self._dump_cached_books(header='after',indent=2)
 
@@ -329,7 +346,8 @@ def books(self, oncard=None, end_session=True):
                              'title':book.Name,
                              'author':book.Artist,
                              'lib_book':library_books[this_book.path] if this_book.path in library_books else None,
-                             'uuid': book.Composer
+                             'uuid': book.Composer,
+                             'format': 'pdf' if book.KindAsString.startswith('PDF') else 'epub'
                              }
 
                             if self.report_progress is not None:
@@ -343,9 +361,9 @@ def books(self, oncard=None, end_session=True):
                 if self.report_progress is not None:
                     self.report_progress(1.0, _('finished'))
                 self.cached_books = cached_books
-#                 if DEBUG:
-#                     self._dump_booklist(booklist, 'returning from books():')
-#                     self._dump_cached_books('returning from books():')
+                if DEBUG:
+                    self._dump_booklist(booklist, 'returning from books()', indent=2)
+                    self._dump_cached_books('returning from books()',indent=2)
                 return booklist
         else:
             return []
@@ -506,6 +524,19 @@ def card_prefix(self, end_session=True):
         '''
         return (None,None)
 
+    @classmethod
+    def config_widget(cls):
+        '''
+        Return a QWidget with settings for the device interface
+        '''
+        cw = DriverBase.config_widget()
+        # Turn off the Save template
+        cw.opt_save_template.setVisible(False)
+        cw.label.setVisible(False)
+        # Repurpose the checkbox
+        cw.opt_read_metadata.setText(_("Use Series as Category in iTunes/iBooks"))
+        return cw
+
     def delete_books(self, paths, end_session=True):
         '''
         Delete books at paths on device.
@@ -685,6 +716,9 @@ def remove_books_from_metadata(self, paths, booklists):
         @param booklists:  A tuple containing the result of calls to
                                 (L{books}(oncard=None), L{books}(oncard='carda'),
                                 L{books}(oncard='cardb')).
+
+        NB: This will not find books that were added by a different installation of calibre
+            as uuids are different
         '''
         if DEBUG:
             self.log.info("ITUNES.remove_books_from_metadata()")
@@ -732,17 +766,6 @@ def set_progress_reporter(self, report_progress):
         '''
         self.report_progress = report_progress
 
-    def settings(self):
-        '''
-        Should return an opts object. The opts object should have one attribute
-        `format_map` which is an ordered list of formats for the device.
-        '''
-        klass = self if isinstance(self, type) else self.__class__
-        c = Config('device_drivers_%s' % klass.__name__, _('settings for device drivers'))
-        c.add_opt('format_map', default=self.FORMATS,
-            help=_('Ordered list of formats the device will accept'))
-        return c.parse()
-
     def sync_booklists(self, booklists, end_session=True):
         '''
         Update metadata on device.
@@ -750,6 +773,10 @@ def sync_booklists(self, booklists, end_session=True):
                                 (L{books}(oncard=None), L{books}(oncard='carda'),
                                 L{books}(oncard='cardb')).
         '''
+
+        if DEBUG:
+            self.log.info("ITUNES.sync_booklists()")
+
         if self.update_needed:
             if DEBUG:
                 self.log.info(' calling _update_device')
@@ -812,29 +839,33 @@ def upload_books(self, files, names, on_card=None, end_session=True,
         self.problem_msg = _("Some cover art could not be converted.\n"
                                      "Click 'Show Details' for a list.")
 
-        if False:
+        if DEBUG:
             self.log.info("ITUNES.upload_books()")
             self._dump_files(files, header='upload_books()',indent=2)
             self._dump_update_list(header='upload_books()',indent=2)
+            #self.log.info("  self.settings().format_map: %s" % self.settings().format_map)
 
         if isosx:
             for (i,file) in enumerate(files):
+                format = file.rpartition('.')[2].lower()
                 path = self.path_template % (metadata[i].title, metadata[i].author[0])
                 self._remove_existing_copy(path, metadata[i])
-                fpath = self._get_fpath(file, metadata[i], update_md=True)
+                fpath = self._get_fpath(file, metadata[i], format, update_md=True)
                 db_added, lb_added = self._add_new_copy(fpath, metadata[i])
-                thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added)
-                this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb)
+                thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
+                this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
                 new_booklist.append(this_book)
                 self._update_iTunes_metadata(metadata[i], db_added, lb_added, this_book)
 
                 # Add new_book to self.cached_paths
                 self.cached_books[this_book.path] = {
-                    'title': metadata[i].title,
                    'author': metadata[i].author,
-                 'lib_book': lb_added,
                  'dev_book': db_added,
-                     'uuid': metadata[i].uuid}
+                   'format': format,
+                 'lib_book': lb_added,
+                    'title': metadata[i].title,
+                     'uuid': metadata[i].uuid }
+
 
                 # Report progress
                 if self.report_progress is not None:
@@ -846,9 +877,10 @@ def upload_books(self, files, names, on_card=None, end_session=True,
                 self.iTunes = win32com.client.Dispatch("iTunes.Application")
 
                 for (i,file) in enumerate(files):
+                    format = file.rpartition('.')[2].lower()
                     path = self.path_template % (metadata[i].title, metadata[i].author[0])
                     self._remove_existing_copy(path, metadata[i])
-                    fpath = self._get_fpath(file, metadata[i], update_md=True)
+                    fpath = self._get_fpath(file, metadata[i],format, update_md=True)
                     db_added, lb_added = self._add_new_copy(fpath, metadata[i])
 
                     if self.manual_sync_mode and not db_added:
@@ -857,17 +889,18 @@ def upload_books(self, files, names, on_card=None, end_session=True,
                                             "Click 'Show Details...' for affected books.")
                         self.problem_titles.append("'%s' by %s" % (metadata[i].title, metadata[i].author[0]))
 
-                    thumb = self._cover_to_thumb(path, metadata[i], lb_added, db_added)
-                    this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb)
+                    thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
+                    this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
                     new_booklist.append(this_book)
                     self._update_iTunes_metadata(metadata[i], db_added, lb_added, this_book)
 
                     # Add new_book to self.cached_paths
                     self.cached_books[this_book.path] = {
-                        'title': metadata[i].title,
                        'author': metadata[i].author[0],
-                     'lib_book': lb_added,
                      'dev_book': db_added,
+                       'format': format,
+                     'lib_book': lb_added,
+                        'title': metadata[i].title,
                          'uuid': metadata[i].uuid}
 
                     # Report progress
@@ -968,7 +1001,8 @@ def _add_device_book(self,fpath, metadata):
                         db_added = self._find_device_book(
                                      {'title': metadata.title,
                                      'author': metadata.authors[0],
-                                       'uuid': metadata.uuid})
+                                       'uuid': metadata.uuid,
+                                     'format': fpath.rpartition('.')[2].lower()})
 
                     return db_added
 
@@ -1021,7 +1055,8 @@ def _add_library_book(self,file, metadata):
                 added = self._find_library_book(
                     { 'title': metadata.title,
                      'author': metadata.author[0],
-                       'uuid': metadata.uuid})
+                       'uuid': metadata.uuid,
+                     'format': file.rpartition('.')[2].lower()})
         return added
 
     def _add_new_copy(self, fpath, metadata):
@@ -1047,46 +1082,82 @@ def _add_new_copy(self, fpath, metadata):
 
         return db_added, lb_added
 
-    def _cover_to_thumb(self, path, metadata, db_added, lb_added):
+    def _cover_to_thumb(self, path, metadata, db_added, lb_added, format):
         '''
         assumes pythoncom wrapper for db_added
+        as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
         '''
         self.log.info(" ITUNES._cover_to_thumb()")
+
         thumb = None
         if metadata.cover:
-            if isosx:
-                cover_data = open(metadata.cover,'rb')
-                if lb_added:
-                    lb_added.artworks[1].data_.set(cover_data.read())
 
-                if db_added:
-                    # The following command generates an error, but the artwork does in fact
-                    # get sent to the device.  Seems like a bug in Apple's automation interface
-                    try:
-                        db_added.artworks[1].data_.set(cover_data.read())
-                    except:
+            if (format == 'epub'):
+                # Pre-shrink cover
+                # self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT
+                try:
+                    img = PILImage.open(metadata.cover)
+                    width = img.size[0]
+                    height = img.size[1]
+                    scaled, nwidth, nheight = fit_image(width, height, self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT)
+                    if scaled:
                         if DEBUG:
-                            self.log.warning("  iTunes automation interface reported an error"
-                                             " when adding artwork to '%s' on the iDevice" % metadata.title)
-                        #import traceback
-                        #traceback.print_exc()
-                        #from calibre import ipython
-                        #ipython(user_ns=locals())
-                        pass
-
-
-            elif iswindows:
-                if lb_added:
-                    if lb_added.Artwork.Count:
-                        lb_added.Artwork.Item(1).SetArtworkFromFile(metadata.cover)
+                            self.log.info("   '%s' scaled from %sx%s to %sx%s" %
+                                          (metadata.cover,width,height,nwidth,nheight))
+                        img = img.resize((nwidth, nheight), PILImage.ANTIALIAS)
+                        cd = cStringIO.StringIO()
+                        img.convert('RGB').save(cd, 'JPEG')
+                        cover_data = cd.getvalue()
+                        cd.close()
                     else:
-                        lb_added.AddArtworkFromFile(metadata.cover)
+                        with open(metadata.cover,'r+b') as cd:
+                            cover_data = cd.read()
+                except:
+                    self.problem_titles.append("'%s' by %s" % (metadata.title, metadata.author[0]))
+                    self.log.error("  error scaling '%s' for '%s'" % (metadata.cover,metadata.title))
+                    return thumb
 
-                if db_added:
-                    if db_added.Artwork.Count:
-                        db_added.Artwork.Item(1).SetArtworkFromFile(metadata.cover)
-                    else:
-                        db_added.AddArtworkFromFile(metadata.cover)
+                if isosx:
+                    if lb_added:
+                        lb_added.artworks[1].data_.set(cover_data)
+
+                    if db_added:
+                        # The following command generates an error, but the artwork does in fact
+                        # get sent to the device.  Seems like a bug in Apple's automation interface
+                        try:
+                            db_added.artworks[1].data_.set(cover_data)
+                        except:
+                            if DEBUG:
+                                self.log.warning("  iTunes automation interface reported an error"
+                                                 " when adding artwork to '%s' on the iDevice" % metadata.title)
+                            #import traceback
+                            #traceback.print_exc()
+                            #from calibre import ipython
+                            #ipython(user_ns=locals())
+                            pass
+
+
+                elif iswindows:
+                    # Write the data to a real file for Windows iTunes
+                    tc = os.path.join(tempfile.gettempdir(), "cover.jpg")
+                    with open(tc,'wb') as tmp_cover:
+                        tmp_cover.write(cover_data)
+
+                    if lb_added:
+                        if lb_added.Artwork.Count:
+                            lb_added.Artwork.Item(1).SetArtworkFromFile(tc)
+                        else:
+                            lb_added.AddArtworkFromFile(tc)
+
+                    if db_added:
+                        if db_added.Artwork.Count:
+                            db_added.Artwork.Item(1).SetArtworkFromFile(tc)
+                        else:
+                            db_added.AddArtworkFromFile(tc)
+
+            elif format == 'pdf':
+                if DEBUG:
+                    self.log.info("   unable to set PDF cover via automation interface")
 
             try:
                 # Resize for thumb
@@ -1097,6 +1168,7 @@ def _cover_to_thumb(self, path, metadata, db_added, lb_added):
                 of = cStringIO.StringIO()
                 im.convert('RGB').save(of, 'JPEG')
                 thumb = of.getvalue()
+                of.close()
 
                 # Refresh the thumbnail cache
                 if DEBUG:
@@ -1105,14 +1177,15 @@ def _cover_to_thumb(self, path, metadata, db_added, lb_added):
                 zfw = ZipFile(archive_path, mode='a')
                 thumb_path = path.rpartition('.')[0] + '.jpg'
                 zfw.writestr(thumb_path, thumb)
-                zfw.close()
             except:
                 self.problem_titles.append("'%s' by %s" % (metadata.title, metadata.author[0]))
                 self.log.error("  error converting '%s' to thumb for '%s'" % (metadata.cover,metadata.title))
+            finally:
+                zfw.close()
 
-            return thumb
+        return thumb
 
-    def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb):
+    def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb, format):
         '''
         '''
         if DEBUG:
@@ -1122,6 +1195,7 @@ def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb):
 
         this_book.db_id = None
         this_book.device_collections = []
+        this_book.format = format
         this_book.library_id = lb_added
         this_book.path = path
         this_book.thumbnail = thumb
@@ -1134,13 +1208,13 @@ def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb):
                 try:
                     this_book.datetime = parse_date(str(lb_added.date_added())).timetuple()
                 except:
-                    pass
+                    this_book.datetime = time.gmtime()
             elif db_added:
                 this_book.size = self._get_device_book_size(fpath, db_added.size())
                 try:
                     this_book.datetime = parse_date(str(db_added.date_added())).timetuple()
                 except:
-                    pass
+                    this_book.datetime = time.gmtime()
 
         elif iswindows:
             if lb_added:
@@ -1148,13 +1222,13 @@ def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb):
                 try:
                     this_book.datetime = parse_date(str(lb_added.DateAdded)).timetuple()
                 except:
-                    pass
+                    this_book.datetime = time.gmtime()
             elif db_added:
                 this_book.size = self._get_device_book_size(fpath, db_added.Size)
                 try:
                     this_book.datetime = parse_date(str(db_added.DateAdded)).timetuple()
                 except:
-                    pass
+                    this_book.datetime = time.gmtime()
 
         return this_book
 
@@ -1319,10 +1393,11 @@ def _dump_cached_books(self, header=None, indent=0):
                   self.cached_books[cb]['uuid']))
         elif iswindows:
             for cb in self.cached_books.keys():
-                self.log.info("%s%-40.40s %-30.30s %s" %
+                self.log.info("%s%-40.40s %-30.30s %-4.4s %s" %
                  (' '*indent,
                   self.cached_books[cb]['title'],
                   self.cached_books[cb]['author'],
+                  self.cached_books[cb]['format'],
                   self.cached_books[cb]['uuid']))
 
         self.log.info()
@@ -1338,8 +1413,9 @@ def _dump_epub_metadata(self, fpath):
         fnames = zf.namelist()
         opf = [x for x in fnames if '.opf' in x][0]
         if opf:
-            opf_raw = cStringIO.StringIO(zf.read(opf)).getvalue()
-            soup = BeautifulSoup(opf_raw)
+            opf_raw = cStringIO.StringIO(zf.read(opf))
+            soup = BeautifulSoup(opf_raw.getvalue())
+            opf_raw.close()
             title = soup.find('dc:title').renderContents()
             author = soup.find('dc:creator').renderContents()
             ts = soup.find('meta',attrs={'name':'calibre:timestamp'})
@@ -1428,7 +1504,7 @@ def _find_device_book(self, search):
                 hits = dev_books.Search(search['uuid'],self.SearchField.index('All'))
                 if hits:
                     hit = hits[0]
-                    self.log.info("  found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
+                    self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
                     return hit
 
                 # Try by author - there could be multiple hits
@@ -1437,9 +1513,25 @@ def _find_device_book(self, search):
                     for hit in hits:
                         if hit.Name == search['title']:
                             if DEBUG:
-                                self.log.info("  found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
+                                self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
                             return hit
 
+                # PDF metadata was rewritten at export as 'safe(title) - safe(author)'
+                if search['format'] == 'pdf':
+                    title = re.sub(r'[^0-9a-zA-Z ]', '_', search['title'])
+                    author = re.sub(r'[^0-9a-zA-Z ]', '_', search['author'])
+                    if DEBUG:
+                        self.log.info("   searching by name: '%s - %s'" % (title,author))
+                    hits = dev_books.Search('%s - %s' % (title,author),
+                                             self.SearchField.index('All'))
+                    if hits:
+                        hit = hits[0]
+                        self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
+                        return hit
+                    else:
+                        if DEBUG:
+                            self.log.info("   no PDF hits")
+
                 attempts -= 1
                 time.sleep(0.5)
                 if DEBUG:
@@ -1496,7 +1588,7 @@ def _find_library_book(self, search):
                     if hits:
                         hit = hits[0]
                         if DEBUG:
-                            self.log.info("  found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
+                            self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
                         return hit
 
                 if DEBUG:
@@ -1506,9 +1598,25 @@ def _find_library_book(self, search):
                     for hit in hits:
                         if hit.Name == search['title']:
                             if DEBUG:
-                                self.log.info("  found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
+                                self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
                             return hit
 
+                # PDF metadata was rewritten at export as 'safe(title) - safe(author)'
+                if search['format'] == 'pdf':
+                    title = re.sub(r'[^0-9a-zA-Z ]', '_', search['title'])
+                    author = re.sub(r'[^0-9a-zA-Z ]', '_', search['author'])
+                    if DEBUG:
+                        self.log.info("   searching by name: %s - %s" % (title,author))
+                    hits = lib_books.Search('%s - %s' % (title,author),
+                                             self.SearchField.index('All'))
+                    if hits:
+                        hit = hits[0]
+                        self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
+                        return hit
+                    else:
+                        if DEBUG:
+                            self.log.info("   no PDF hits")
+
                 attempts -= 1
                 time.sleep(0.5)
                 if DEBUG:
@@ -1523,10 +1631,12 @@ def _generate_thumbnail(self, book_path, book):
         Convert iTunes artwork to thumbnail
         Cache generated thumbnails
         cache_dir = os.path.join(config_dir, 'caches', 'itunes')
+        as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
         '''
 
         archive_path = os.path.join(self.cache_dir, "thumbs.zip")
         thumb_path = book_path.rpartition('.')[0] + '.jpg'
+        format = book_path.rpartition('.')[2].lower()
 
         try:
             zfr = ZipFile(archive_path)
@@ -1539,77 +1649,99 @@ def _generate_thumbnail(self, book_path, book):
 
         self.log.info(" ITUNES._generate_thumbnail()")
         if isosx:
-            try:
-                # Resize the cover
-                data = book.artworks[1].raw_data().data
-                #self._dump_hex(data[:256])
-                im = PILImage.open(cStringIO.StringIO(data))
-                scaled, width, height = fit_image(im.size[0],im.size[1], 60, 80)
-                im = im.resize((int(width),int(height)), PILImage.ANTIALIAS)
-                thumb = cStringIO.StringIO()
-                im.convert('RGB').save(thumb,'JPEG')
-
-                # Cache the tagged thumb
-                if DEBUG:
-                    self.log.info("  generated thumb for '%s', caching" % book.name())
-                zfw.writestr(thumb_path, thumb.getvalue())
-                zfw.close()
-                return thumb.getvalue()
-            except:
-                self.log.error("  error generating thumb for '%s'" % book.name())
+            if format == 'epub':
                 try:
+                    if False:
+                        self.log.info("   fetching artwork from %s\n   %s" % (book_path,book))
+                    # Resize the cover
+                    data = book.artworks[1].raw_data().data
+                    #self._dump_hex(data[:256])
+                    img_data = cStringIO.StringIO(data)
+                    im = PILImage.open(img_data)
+                    scaled, width, height = fit_image(im.size[0],im.size[1], 60, 80)
+                    im = im.resize((int(width),int(height)), PILImage.ANTIALIAS)
+                    img_data.close()
+
+                    thumb = cStringIO.StringIO()
+                    im.convert('RGB').save(thumb,'JPEG')
+                    thumb_data = thumb.getvalue()
+                    thumb.close()
+
+                    # Cache the tagged thumb
+                    if DEBUG:
+                        self.log.info("  generated thumb for '%s', caching" % book.name())
+                    zfw.writestr(thumb_path, thumb_data)
                     zfw.close()
+                    return thumb_data
                 except:
-                    pass
+                    self.log.error("  error generating thumb for '%s'" % book.name())
+                    try:
+                        zfw.close()
+                    except:
+                        pass
+                    return None
+            else:
+                if DEBUG:
+                    self.log.info("  unable to generate PDF thumbs")
                 return None
 
         elif iswindows:
 
             if not book.Artwork.Count:
                 if DEBUG:
-                    self.log.info("  no artwork available")
+                    self.log.info("  no artwork available for '%s'" % book.Name)
                 return None
 
-            # Save the cover from iTunes
-            try:
-                tmp_thumb = os.path.join(tempfile.gettempdir(), "thumb.%s" % self.ArtworkFormat[book.Artwork.Item(1).Format])
-                book.Artwork.Item(1).SaveArtworkToFile(tmp_thumb)
-                # Resize the cover
-                im = PILImage.open(tmp_thumb)
-                scaled, width, height = fit_image(im.size[0],im.size[1], 60, 80)
-                im = im.resize((int(width),int(height)), PILImage.ANTIALIAS)
-                thumb = cStringIO.StringIO()
-                im.convert('RGB').save(thumb,'JPEG')
-                os.remove(tmp_thumb)
-
-                # Cache the tagged thumb
-                if DEBUG:
-                    self.log.info("  generated thumb for '%s', caching" % book.Name)
-                zfw.writestr(thumb_path, thumb.getvalue())
-                zfw.close()
-                return thumb.getvalue()
-            except:
-                self.log.error("  error generating thumb for '%s'" % book.Name)
+            if format == 'epub':
+                # Save the cover from iTunes
                 try:
+                    tmp_thumb = os.path.join(tempfile.gettempdir(), "thumb.%s" % self.ArtworkFormat[book.Artwork.Item(1).Format])
+                    book.Artwork.Item(1).SaveArtworkToFile(tmp_thumb)
+                    # Resize the cover
+                    im = PILImage.open(tmp_thumb)
+                    scaled, width, height = fit_image(im.size[0],im.size[1], 60, 80)
+                    im = im.resize((int(width),int(height)), PILImage.ANTIALIAS)
+                    thumb = cStringIO.StringIO()
+                    im.convert('RGB').save(thumb,'JPEG')
+                    thumb_data = thumb.getvalue()
+                    os.remove(tmp_thumb)
+                    thumb.close()
+
+                    # Cache the tagged thumb
+                    if DEBUG:
+                        self.log.info("  generated thumb for '%s', caching" % book.Name)
+                    zfw.writestr(thumb_path, thumb_data)
                     zfw.close()
+                    return thumb_data
                 except:
-                    pass
+                    self.log.error("  error generating thumb for '%s'" % book.Name)
+                    try:
+                        zfw.close()
+                    except:
+                        pass
+                    return None
+            else:
+                if DEBUG:
+                    self.log.info("  unable to generate PDF thumbs")
                 return None
 
     def _get_device_book_size(self, file, compressed_size):
         '''
         Calculate the exploded size of file
         '''
-        myZip = ZipFile(file,'r')
-        myZipList = myZip.infolist()
-        exploded_file_size = 0
-        for file in myZipList:
-            exploded_file_size += file.file_size
-        if False:
-            self.log.info(" ITUNES._get_device_book_size()")
-            self.log.info("  %d items in archive" % len(myZipList))
-            self.log.info("  compressed: %d  exploded: %d" % (compressed_size, exploded_file_size))
-        myZip.close()
+        exploded_file_size = compressed_size
+        format = file.rpartition('.')[2].lower()
+        if format == 'epub':
+            myZip = ZipFile(file,'r')
+            myZipList = myZip.infolist()
+            exploded_file_size = 0
+            for file in myZipList:
+                exploded_file_size += file.file_size
+            if False:
+                self.log.info(" ITUNES._get_device_book_size()")
+                self.log.info("  %d items in archive" % len(myZipList))
+                self.log.info("  compressed: %d  exploded: %d" % (compressed_size, exploded_file_size))
+            myZip.close()
         return exploded_file_size
 
     def _get_device_books(self):
@@ -1701,7 +1833,7 @@ def _get_device_books_playlist(self):
                         self.log.error("  no iPad|Books playlist found")
                 return pl
 
-    def _get_fpath(self,file, metadata, update_md=False):
+    def _get_fpath(self,file, metadata, format, update_md=False):
         '''
         If the database copy will be deleted after upload, we have to
         use file (the PersistentTemporaryFile), which will be around until
@@ -1723,9 +1855,9 @@ def _get_fpath(self,file, metadata, update_md=False):
         else:
             # Recipe - PTF
             if DEBUG:
-                self.log.info("  file will be deleted after upload")
+                self.log.info("   file will be deleted after upload")
 
-        if update_md:
+        if format == 'epub' and update_md:
             self._update_epub_metadata(fpath, metadata)
 
         return fpath
@@ -1775,10 +1907,14 @@ def _get_library_books(self):
                             # Collect calibre orphans - remnants of recipe uploads
                             path = self.path_template % (book.name(), book.artist())
                             if str(book.description()).startswith(self.description_prefix):
-                                if book.location() == appscript.k.missing_value:
-                                    library_orphans[path] = book
-                                    if False:
-                                        self.log.info("   found iTunes PTF '%s' in Library|Books" % book.name())
+                                try:
+                                    if book.location() == appscript.k.missing_value:
+                                        library_orphans[path] = book
+                                        if False:
+                                            self.log.info("   found iTunes PTF '%s' in Library|Books" % book.name())
+                                except:
+                                    if DEBUG:
+                                        self.log.error("   iTunes returned an error returning .location() with %s" % book.name())
 
                             library_books[path] = book
                             if DEBUG:
@@ -1937,10 +2073,22 @@ def _launch_iTunes(self):
                  (self.iTunes.name(), self.iTunes.version(), self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
                 self.log.info("  iTunes_media: %s" % self.iTunes_media)
+
         if iswindows:
             '''
             Launch iTunes if not already running
             Assumes pythoncom wrapper
+
+            *** Current implementation doesn't handle UNC paths correctly,
+                and python has two incompatible methods to parse UNCs:
+                os.path.splitdrive() and os.path.splitunc()
+                need to use os.path.normpath on result of splitunc()
+
+                Once you have the //server/share, convert with os.path.normpath('//server/share')
+                os.path.splitdrive doesn't work as advertised, so use os.path.splitunc
+                os.path.splitunc("//server/share") returns ('//server/share','')
+                os.path.splitunc("C:/Documents") returns ('c:','/documents')
+                os.path.normpath("//server/share") returns "\\\\server\\share"
             '''
             # Instantiate iTunes
             self.iTunes = win32com.client.Dispatch("iTunes.Application")
@@ -1949,17 +2097,23 @@ def _launch_iTunes(self):
             self.initial_status = 'launched'
 
             # Read the current storage path for iTunes media from the XML file
+            media_dir = ''
+            string = None
             with open(self.iTunes.LibraryXMLPath, 'r') as xml:
-                soup = BeautifulSoup(xml.read().decode('utf-8'))
-                mf = soup.find('key',text="Music Folder").parent
-                string = mf.findNext('string').renderContents()
-                media_dir = os.path.abspath(string[len('file://localhost/'):].replace('%20',' '))
+                for line in xml:
+                    if line.strip().startswith('<key>Music Folder'):
+                        soup = BeautifulSoup(line)
+                        string = soup.find('string').renderContents()
+                        media_dir = os.path.abspath(string[len('file://localhost/'):].replace('%20',' '))
+                        break
                 if os.path.exists(media_dir):
                     self.iTunes_media = media_dir
-                else:
+                elif hasattr(string,'parent'):
                     self.log.error("  could not extract valid iTunes.media_dir from %s" % self.iTunes.LibraryXMLPath)
                     self.log.error("  %s" % string.parent.prettify())
                     self.log.error("  '%s' not found" % media_dir)
+                else:
+                    self.log.error("  no media dir found: string: %s" % string)
 
             if DEBUG:
                 self.log.info("  %s %s" % (__appname__, __version__))
@@ -2028,7 +2182,9 @@ def _remove_existing_copy(self, path, metadata):
             # Delete existing from Library|Books, add to self.update_list
             # for deletion from booklist[0] during add_books_to_metadata
             for book in self.cached_books:
-                if self.cached_books[book]['uuid'] == metadata.uuid:
+                if (self.cached_books[book]['uuid'] == metadata.uuid) or \
+                   (self.cached_books[book]['title'] == metadata.title and \
+                    self.cached_books[book]['author'] == metadata.authors[0]):
                     self.update_list.append(self.cached_books[book])
                     self._remove_from_iTunes(self.cached_books[book])
                     if DEBUG:
@@ -2036,7 +2192,7 @@ def _remove_existing_copy(self, path, metadata):
                     break
                 else:
                     if DEBUG:
-                        self.log.info("  '%s' not in cached_books" % metadata.title)
+                        self.log.info("  '%s' not found in cached_books" % metadata.title)
 
     def _remove_from_device(self, cached_book):
         '''
@@ -2119,8 +2275,8 @@ def _remove_from_iTunes(self, cached_book):
                 path = book.Location
 
             if book:
-                storage_path = os.path.split(path)
-                if path.startswith(self.iTunes_media):
+                if self.iTunes_media and path.startswith(self.iTunes_media):
+                    storage_path = os.path.split(path)
                     if DEBUG:
                         self.log.info("   removing '%s' at %s" %
                             (cached_book['title'], path))
@@ -2158,20 +2314,34 @@ def _update_epub_metadata(self, fpath, metadata):
             fnames = zf_opf.namelist()
             opf = [x for x in fnames if '.opf' in x][0]
             if opf:
-                opf_raw = cStringIO.StringIO(zf_opf.read(opf)).getvalue()
-                soup = BeautifulSoup(opf_raw)
+                opf_raw = cStringIO.StringIO(zf_opf.read(opf))
+                soup = BeautifulSoup(opf_raw.getvalue())
+                opf_raw.close()
+
+                # Touch existing calibre timestamp
                 md = soup.find('metadata')
-                ts = md.find('meta',attrs={'name':'calibre:timestamp'})
-                if ts:
-                    # Touch existing calibre timestamp
-                    timestamp = ts['content']
-                    old_ts = parse_date(timestamp)
-                    metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour,
-                                               old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo)
+                if md:
+                    ts = md.find('meta',attrs={'name':'calibre:timestamp'})
+                    if ts:
+                        timestamp = ts['content']
+                        old_ts = parse_date(timestamp)
+                        metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour,
+                                                   old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo)
+                    else:
+                        metadata.timestamp = isoformat(now())
+                        if DEBUG:
+                            self.log.info("   add timestamp: %s" % metadata.timestamp)
                 else:
                     metadata.timestamp = isoformat(now())
                     if DEBUG:
+                        self.log.warning("   missing <metadata> block in OPF file")
                         self.log.info("   add timestamp: %s" % metadata.timestamp)
+
+                # Force the language declaration for iBooks 1.1
+                metadata.language = get_lang()
+                if DEBUG:
+                    self.log.info("   rewriting language: <dc:language>%s</dc:language>" % metadata.language)
+
             zf_opf.close()
 
             # If 'News' in tags, tweak the title/author for friendlier display in iBooks
@@ -2257,6 +2427,9 @@ def _update_iTunes_metadata(self, metadata, db_added, lb_added, this_book):
                 lb_added.enabled.set(True)
                 lb_added.sort_artist.set(metadata.author_sort.title())
                 lb_added.sort_name.set(this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    lb_added.artist.set(metadata.authors[0])
+                    lb_added.name.set(metadata.title)
 
             if db_added:
                 db_added.album.set(metadata.title)
@@ -2265,6 +2438,9 @@ def _update_iTunes_metadata(self, metadata, db_added, lb_added, this_book):
                 db_added.enabled.set(True)
                 db_added.sort_artist.set(metadata.author_sort.title())
                 db_added.sort_name.set(this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    db_added.artist.set(metadata.authors[0])
+                    db_added.name.set(metadata.title)
 
             if metadata.comments:
                 if lb_added:
@@ -2284,7 +2460,9 @@ def _update_iTunes_metadata(self, metadata, db_added, lb_added, this_book):
 
             # Set genre from series if available, else first alpha tag
             # Otherwise iTunes grabs the first dc:subject from the opf metadata
-            if self.use_series_data and metadata.series:
+            if metadata.series and self.settings().read_metadata:
+                if DEBUG:
+                    self.log.info("   using Series name as Genre")
                 if lb_added:
                     lb_added.sort_name.set("%s %03d" % (metadata.series, metadata.series_index))
                     lb_added.genre.set(metadata.series)
@@ -2298,6 +2476,9 @@ def _update_iTunes_metadata(self, metadata, db_added, lb_added, this_book):
                     db_added.episode_number.set(metadata.series_index)
 
             elif metadata.tags:
+                if DEBUG:
+                    self.log.info("   %susing Tag as Genre" %
+                                  "no Series name available, " if self.settings().read_metadata else '')
                 for tag in metadata.tags:
                     if self._is_alpha(tag[0]):
                         if lb_added:
@@ -2314,6 +2495,9 @@ def _update_iTunes_metadata(self, metadata, db_added, lb_added, this_book):
                 lb_added.Enabled = True
                 lb_added.SortArtist = (metadata.author_sort.title())
                 lb_added.SortName = (this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    lb_added.Artist = metadata.authors[0]
+                    lb_added.Name = metadata.title
 
             if db_added:
                 db_added.Album = metadata.title
@@ -2322,6 +2506,9 @@ def _update_iTunes_metadata(self, metadata, db_added, lb_added, this_book):
                 db_added.Enabled = True
                 db_added.SortArtist = (metadata.author_sort.title())
                 db_added.SortName = (this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    db_added.Artist = metadata.authors[0]
+                    db_added.Name = metadata.title
 
             if metadata.comments:
                 if lb_added:
@@ -2345,7 +2532,9 @@ def _update_iTunes_metadata(self, metadata, db_added, lb_added, this_book):
             # Otherwise iBooks uses first <dc:subject> from opf
             # iTunes balks on setting EpisodeNumber, but it sticks (9.1.1.12)
 
-            if self.use_series_data and metadata.series:
+            if metadata.series and self.settings().read_metadata:
+                if DEBUG:
+                    self.log.info("   using Series name as Genre")
                 if lb_added:
                     lb_added.SortName = "%s %03d" % (metadata.series, metadata.series_index)
                     lb_added.Genre = metadata.series
@@ -2365,6 +2554,8 @@ def _update_iTunes_metadata(self, metadata, db_added, lb_added, this_book):
                             self.log.warning("  iTunes automation interface reported an error"
                                              " setting EpisodeNumber on iDevice")
             elif metadata.tags:
+                if DEBUG:
+                    self.log.info("   using Tag as Genre")
                 for tag in metadata.tags:
                     if self._is_alpha(tag[0]):
                         if lb_added:
@@ -2429,8 +2620,6 @@ def get_collections(self, collection_attributes):
 class Book(MetaInformation):
     '''
     A simple class describing a book in the iTunes Books Library.
-    Q's:
-    - Should thumbnail come from calibre if available?
     - See ebooks.metadata.__init__ for all fields
     '''
     def __init__(self,title,author):

src/calibre/devices/interface.py

@@ -59,7 +59,7 @@ def get_gui_name(cls):
             return cls.__name__
         return cls.name
 
-
+    # Device detection {{{
     def test_bcd_windows(self, device_id, bcd):
         if bcd is None or len(bcd) == 0:
             return True
@@ -152,6 +152,7 @@ def is_usb_connected(self, devices_on_system, debug=False,
                                     return True, dev
         return False, None
 
+    # }}}
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None) :
@@ -372,14 +373,12 @@ def save_settings(cls, settings_widget):
     @classmethod
     def settings(cls):
         '''
-        Should return an opts object. The opts object should have one attribute
+        Should return an opts object. The opts object should have at least one attribute
         `format_map` which is an ordered list of formats for the device.
         '''
         raise NotImplementedError()
 
 
-
-
 class BookList(list):
     '''
     A list of books. Each Book object must have the fields:

src/calibre/devices/kindle/driver.py

@@ -429,6 +429,7 @@ def get_topaz_highlight(displayed_location):
                 entries, = unpack('>I', data[9:13])
                 current_entry = 0
                 e_base = 0x0d
+                self.pdf_page_offset = 0
                 while current_entry < entries:
                     '''
                     location, = unpack('>I', data[e_base+2:e_base+6])

src/calibre/devices/prs505/driver.py

@@ -99,7 +99,7 @@ def write_cache(prefix):
         if self._card_b_prefix is not None:
             if not write_cache(self._card_b_prefix):
                 self._card_b_prefix = None
-
+        self.booklist_class.rebuild_collections = self.rebuild_collections
 
     def get_device_information(self, end_session=True):
         return (self.gui_name, '', '', '')
@@ -145,7 +145,7 @@ def sync_booklists(self, booklists, end_session=True):
                 blists[i] = booklists[i]
         opts = self.settings()
         if opts.extra_customization:
-            collections = [x.strip() for x in
+            collections = [x.lower().strip() for x in
                     opts.extra_customization.split(',')]
         else:
             collections = []
@@ -156,4 +156,10 @@ def sync_booklists(self, booklists, end_session=True):
         USBMS.sync_booklists(self, booklists, end_session=end_session)
         debug_print('PRS505: finished sync_booklists')
 
+    def rebuild_collections(self, booklist, oncard):
+        debug_print('PRS505: started rebuild_collections on card', oncard)
+        c = self.initialize_XML_cache()
+        c.rebuild_collections(booklist, {'carda':1, 'cardb':2}.get(oncard, 0))
+        c.write()
+        debug_print('PRS505: finished rebuild_collections')
 

src/calibre/devices/prs505/sony_cache.py

@@ -6,10 +6,8 @@
 __docformat__ = 'restructuredtext en'
 
 import os, time
-from pprint import pprint
 from base64 import b64decode
 from uuid import uuid4
-
 from lxml import etree
 
 from calibre import prints, guess_type
@@ -62,8 +60,7 @@ class XMLCache(object):
 
     def __init__(self, paths, prefixes, use_author_sort):
         if DEBUG:
-            debug_print('Building XMLCache...')
-            pprint(paths)
+            debug_print('Building XMLCache...', paths)
         self.paths = paths
         self.prefixes = prefixes
         self.use_author_sort = use_author_sort
@@ -151,15 +148,14 @@ def ensure_unique_playlist_titles(self):
                 else:
                     seen.add(title)
 
-    def get_playlist_map(self):
-        debug_print('Start get_playlist_map')
+    def build_playlist_id_map(self):
+        debug_print('Start build_playlist_id_map')
         ans = {}
         self.ensure_unique_playlist_titles()
         debug_print('after ensure_unique_playlist_titles')
         self.prune_empty_playlists()
-        debug_print('get_playlist_map loop')
         for i, root in self.record_roots.items():
-            debug_print('get_playlist_map loop', i)
+            debug_print('build_playlist_id_map loop', i)
             id_map = self.build_id_map(root)
             ans[i] = []
             for playlist in root.xpath('//*[local-name()="playlist"]'):
@@ -170,9 +166,23 @@ def get_playlist_map(self):
                     if record is not None:
                         items.append(record)
                 ans[i].append((playlist.get('title'), items))
-        debug_print('end get_playlist_map')
+        debug_print('end build_playlist_id_map')
         return ans
 
+    def build_id_playlist_map(self, bl_index):
+        debug_print('Start build_id_playlist_map')
+        pmap = self.build_playlist_id_map()[bl_index]
+        playlist_map = {}
+        for title, records in pmap:
+            for record in records:
+                path = record.get('path', None)
+                if path:
+                    if path not in playlist_map:
+                        playlist_map[path] = []
+                    playlist_map[path].append(title)
+        debug_print('Finish build_id_playlist_map. Found', len(playlist_map))
+        return playlist_map
+
     def get_or_create_playlist(self, bl_idx, title):
         root = self.record_roots[bl_idx]
         for playlist in root.xpath('//*[local-name()="playlist"]'):
@@ -192,8 +202,7 @@ def get_or_create_playlist(self, bl_idx, title):
     # }}}
 
     def fix_ids(self): # {{{
-        if DEBUG:
-            debug_print('Running fix_ids()')
+        debug_print('Running fix_ids()')
 
         def ensure_numeric_ids(root):
             idmap = {}
@@ -276,38 +285,19 @@ def rebase_ids(root, base, sourceid, pl_sourceid):
     def update_booklist(self, bl, bl_index):
         if bl_index not in self.record_roots:
             return
-        if DEBUG:
-            debug_print('Updating JSON cache:', bl_index)
+        debug_print('Updating JSON cache:', bl_index)
+        playlist_map = self.build_id_playlist_map(bl_index)
         root = self.record_roots[bl_index]
-        pmap = self.get_playlist_map()[bl_index]
-        playlist_map = {}
-        for title, records in pmap:
-            for record in records:
-                path = record.get('path', None)
-                if path:
-                    if path not in playlist_map:
-                        playlist_map[path] = []
-                    playlist_map[path].append(title)
-
         lpath_map = self.build_lpath_map(root)
         for book in bl:
             record = lpath_map.get(book.lpath, None)
             if record is not None:
                 title = record.get('title', None)
                 if title is not None and title != book.title:
-                    if DEBUG:
-                        debug_print('Renaming title', book.title, 'to', title)
+                    debug_print('Renaming title', book.title, 'to', title)
                     book.title = title
-# We shouldn't do this for Sonys, because the reader strips
-# all but the first author.
-#                authors = record.get('author', None)
-#                if authors is not None:
-#                    authors = string_to_authors(authors)
-#                    if authors != book.authors:
-#                        if DEBUG:
-#                            prints('Renaming authors', book.authors, 'to',
-#                                    authors)
-#                        book.authors = authors
+                    # Don't set the author, because the reader strips all but
+                    # the first author.
                 for thumbnail in record.xpath(
                         'descendant::*[local-name()="thumbnail"]'):
                     for img in thumbnail.xpath(
@@ -318,45 +308,50 @@ def update_booklist(self, bl, bl_index):
                             book.thumbnail = raw
                             break
                     break
-                if book.lpath in playlist_map:
-                    tags = playlist_map[book.lpath]
-                    book.device_collections = tags
+                book.device_collections = playlist_map.get(book.lpath, [])
         debug_print('Finished updating JSON cache:', bl_index)
 
     # }}}
 
     # Update XML from JSON {{{
     def update(self, booklists, collections_attributes):
-        debug_print('Starting update XML from JSON')
-        playlist_map = self.get_playlist_map()
-
+        debug_print('Starting update', collections_attributes)
         for i, booklist in booklists.items():
-            if DEBUG:
-                debug_print('Updating XML Cache:', i)
+            playlist_map = self.build_id_playlist_map(i)
+            debug_print('Updating XML Cache:', i)
             root = self.record_roots[i]
             lpath_map = self.build_lpath_map(root)
             for book in booklist:
                 path = os.path.join(self.prefixes[i], *(book.lpath.split('/')))
-#                record = self.book_by_lpath(book.lpath, root)
                 record = lpath_map.get(book.lpath, None)
                 if record is None:
                     record = self.create_text_record(root, i, book.lpath)
                 self.update_text_record(record, book, path, i)
+                # Ensure the collections in the XML database are recorded for
+                # this book
+                if book.device_collections is None:
+                    book.device_collections = []
+                book.device_collections = playlist_map.get(book.lpath, [])
+            self.update_playlists(i, root, booklist, collections_attributes)
+        # Update the device collections because update playlist could have added
+        # some new ones.
+        debug_print('In update/ Starting refresh of device_collections')
+        for i, booklist in booklists.items():
+            playlist_map = self.build_id_playlist_map(i)
+            for book in booklist:
+                book.device_collections = playlist_map.get(book.lpath, [])
+        self.fix_ids()
+        debug_print('Finished update')
 
-            bl_pmap = playlist_map[i]
-            self.update_playlists(i, root, booklist, bl_pmap,
-                    collections_attributes)
-
+    def rebuild_collections(self, booklist, bl_index):
+        if bl_index not in self.record_roots:
+            return
+        root = self.record_roots[bl_index]
+        self.update_playlists(bl_index, root, booklist, [])
         self.fix_ids()
 
-        # This is needed to update device_collections
-        for i, booklist in booklists.items():
-            self.update_booklist(booklist, i)
-        debug_print('Finished update XML from JSON')
-
-    def update_playlists(self, bl_index, root, booklist, playlist_map,
-            collections_attributes):
-        debug_print('Starting update_playlists')
+    def update_playlists(self, bl_index, root, booklist, collections_attributes):
+        debug_print('Starting update_playlists', collections_attributes, bl_index)
         collections = booklist.get_collections(collections_attributes)
         lpath_map = self.build_lpath_map(root)
         for category, books in collections.items():
@@ -372,10 +367,8 @@ def update_playlists(self, bl_index, root, booklist, playlist_map,
                     rec.set('id', str(self.max_id(root)+1))
             ids = [x.get('id', None) for x in records]
             if None in ids:
-                if DEBUG:
-                    debug_print('WARNING: Some <text> elements do not have ids')
-                    ids = [x for x in ids if x is not None]
-
+                debug_print('WARNING: Some <text> elements do not have ids')
+                ids = [x for x in ids if x is not None]
             playlist = self.get_or_create_playlist(bl_index, category)
             playlist_ids = []
             for item in playlist:
@@ -544,10 +537,5 @@ def detect_namespaces(self):
                         break
                 self.namespaces[i] = ns
 
-#        if DEBUG:
-#            debug_print('Found nsmaps:')
-#            pprint(self.nsmaps)
-#            debug_print('Found namespaces:')
-#            pprint(self.namespaces)
     # }}}
 

src/calibre/devices/usbms/books.py

@@ -11,6 +11,7 @@
 from calibre.devices.interface import BookList as _BookList
 from calibre.constants import filesystem_encoding, preferred_encoding
 from calibre import isbytestring
+from calibre.utils.config import prefs
 
 class Book(MetaInformation):
 
@@ -76,7 +77,7 @@ def smart_update(self, other):
         in C{other} takes precedence, unless the information in C{other} is NULL.
         '''
 
-        MetaInformation.smart_update(self, other)
+        MetaInformation.smart_update(self, other, replace_tags=True)
 
         for attr in self.BOOK_ATTRS:
             if hasattr(other, attr):
@@ -132,7 +133,9 @@ def supports_collections(self):
     def get_collections(self, collection_attributes):
         collections = {}
         series_categories = set([])
-        collection_attributes = list(collection_attributes)+['device_collections']
+        collection_attributes = list(collection_attributes)
+        if prefs['preserve_user_collections']:
+            collection_attributes += ['device_collections']
         for attr in collection_attributes:
             attr = attr.strip()
             for book in self:
@@ -167,3 +170,15 @@ def getter(x):
                 books.sort(cmp=lambda x,y:cmp(getter(x), getter(y)))
         return collections
 
+    def rebuild_collections(self, booklist, oncard):
+        '''
+        For each book in the booklist for the card oncard, remove it from all
+        its current collections, then add it to the collections specified in
+        device_collections.
+
+        oncard is None for the main memory, carda for card A, cardb for card B,
+        etc.
+
+        booklist is the object created by the :method:`books` call above.
+        '''
+        pass

src/calibre/devices/usbms/device.py

@@ -78,9 +78,6 @@ class Device(DeviceConfig, DevicePlugin):
     STORAGE_CARD_VOLUME_LABEL = ''
     STORAGE_CARD2_VOLUME_LABEL = None
 
-    SUPPORTS_SUB_DIRS = False
-    MUST_READ_METADATA = False
-    SUPPORTS_USE_AUTHOR_SORT = False
 
     EBOOK_DIR_MAIN = ''
     EBOOK_DIR_CARD_A = ''

src/calibre/devices/usbms/deviceconfig.py

@@ -13,6 +13,10 @@ class DeviceConfig(object):
     EXTRA_CUSTOMIZATION_MESSAGE = None
     EXTRA_CUSTOMIZATION_DEFAULT = None
 
+    SUPPORTS_SUB_DIRS = False
+    MUST_READ_METADATA = False
+    SUPPORTS_USE_AUTHOR_SORT = False
+
     #: If None the default is used
     SAVE_TEMPLATE = None
 
@@ -23,9 +27,14 @@ def _default_save_template(cls):
             config().parse().send_template
 
     @classmethod
-    def _config(cls):
+    def _config_base_name(cls):
         klass = cls if isinstance(cls, type) else cls.__class__
-        c = Config('device_drivers_%s' % klass.__name__, _('settings for device drivers'))
+        return klass.__name__
+
+    @classmethod
+    def _config(cls):
+        name = cls._config_base_name()
+        c = Config('device_drivers_%s' % name, _('settings for device drivers'))
         c.add_opt('format_map', default=cls.FORMATS,
                 help=_('Ordered list of formats the device will accept'))
         c.add_opt('use_subdirs', default=True,

src/calibre/ebooks/conversion/preprocess.py

@@ -107,9 +107,21 @@ class CSSPreProcessor(object):
 
     PAGE_PAT   = re.compile(r'@page[^{]*?{[^}]*?}')
 
-    def __call__(self, data):
+    def __call__(self, data, add_namespace=False):
+        from calibre.ebooks.oeb.base import XHTML_CSS_NAMESPACE
         data = self.PAGE_PAT.sub('', data)
-        return data
+        if not add_namespace:
+            return data
+        ans, namespaced = [], False
+        for line in data.splitlines():
+            ll = line.lstrip()
+            if not (namespaced or ll.startswith('@import') or
+                        ll.startswith('@charset')):
+                ans.append(XHTML_CSS_NAMESPACE.strip())
+                namespaced = True
+            ans.append(line)
+
+        return u'\n'.join(ans)
 
 class HTMLPreProcessor(object):
 

src/calibre/ebooks/html/input.py

@@ -20,7 +20,7 @@
 from calibre.customize.conversion import InputFormatPlugin
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.customize.conversion import OptionRecommendation
-from calibre.constants import islinux, isfreebsd
+from calibre.constants import islinux, isfreebsd, iswindows
 from calibre import unicode_path
 from calibre.utils.localization import get_lang
 from calibre.utils.filenames import ascii_filename
@@ -32,9 +32,14 @@ class Link(object):
 
     @classmethod
     def url_to_local_path(cls, url, base):
-        path = urlunparse(('', '', url.path, url.params, url.query, ''))
+        path = url.path
+        isabs = False
+        if iswindows and path.startswith('/'):
+            path = path[1:]
+            isabs = True
+        path = urlunparse(('', '', path, url.params, url.query, ''))
         path = unquote(path)
-        if os.path.isabs(path):
+        if isabs or os.path.isabs(path):
             return path
         return os.path.abspath(os.path.join(base, path))
 
@@ -307,6 +312,7 @@ def create_oebbook(self, htmlpath, basedir, opts, log, mi):
             xpath
         from calibre import guess_type
         import cssutils
+        self.OEB_STYLES = OEB_STYLES
         oeb = create_oebbook(log, None, opts, self,
                 encoding=opts.input_encoding, populate=False)
         self.oeb = oeb
@@ -371,7 +377,7 @@ def create_oebbook(self, htmlpath, basedir, opts, log, mi):
             rewrite_links(item.data, partial(self.resource_adder, base=dpath))
 
         for item in oeb.manifest.values():
-            if item.media_type in OEB_STYLES:
+            if item.media_type in self.OEB_STYLES:
                 dpath = None
                 for path, href in self.added_resources.items():
                     if href == item.href:
@@ -409,12 +415,30 @@ def create_oebbook(self, htmlpath, basedir, opts, log, mi):
         oeb.container = DirContainer(os.getcwdu(), oeb.log)
         return oeb
 
+    def link_to_local_path(self, link_, base=None):
+        if not isinstance(link_, unicode):
+            try:
+                link_ = link_.decode('utf-8', 'error')
+            except:
+                self.log.warn('Failed to decode link %r. Ignoring'%link_)
+                return None, None
+        try:
+            l = Link(link_, base if base else os.getcwdu())
+        except:
+            self.log.exception('Failed to process link: %r'%link_)
+            return None, None
+        if l.path is None:
+            # Not a local resource
+            return None, None
+        link = l.path.replace('/', os.sep).strip()
+        frag = l.fragment
+        if not link:
+            return None, None
+        return link, frag
 
     def resource_adder(self, link_, base=None):
-        link = self.urlnormalize(link_)
-        link, frag = self.urldefrag(link)
-        link = unquote(link).replace('/', os.sep)
-        if not link.strip():
+        link, frag = self.link_to_local_path(link_, base=base)
+        if link is None:
             return link_
         try:
             if base and not os.path.isabs(link):
@@ -442,6 +466,9 @@ def resource_adder(self, link_, base=None):
 
             item = self.oeb.manifest.add(id, href, media_type)
             item.html_input_href = bhref
+            if guessed in self.OEB_STYLES:
+                item.override_css_fetch = partial(
+                        self.css_import_handler, os.path.dirname(link))
             item.data
             self.added_resources[link] = href
 
@@ -450,7 +477,17 @@ def resource_adder(self, link_, base=None):
             nlink = '#'.join((nlink, frag))
         return nlink
 
-
+    def css_import_handler(self, base, href):
+        link, frag = self.link_to_local_path(href, base=base)
+        if link is None or not os.access(link, os.R_OK) or os.path.isdir(link):
+            return (None, None)
+        try:
+            raw = open(link, 'rb').read().decode('utf-8', 'replace')
+            raw = self.oeb.css_preprocessor(raw, add_namespace=True)
+        except:
+            self.log.exception('Failed to read CSS file: %r'%link)
+            return (None, None)
+        return (None, raw)
 
 
 

src/calibre/ebooks/metadata/__init__.py

@@ -268,7 +268,7 @@ def print_all_attributes(self):
                   ):
             prints(x, getattr(self, x, 'None'))
 
-    def smart_update(self, mi):
+    def smart_update(self, mi, replace_tags=False):
         '''
         Merge the information in C{mi} into self. In case of conflicts, the information
         in C{mi} takes precedence, unless the information in mi is NULL.
@@ -282,7 +282,7 @@ def smart_update(self, mi):
         for attr in ('author_sort', 'title_sort', 'category',
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'application_id', 'manifest', 'spine', 'toc',
-                     'cover', 'language', 'guide', 'book_producer',
+                     'cover', 'guide', 'book_producer',
                      'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights',
                      'publication_type', 'uuid'):
             if hasattr(mi, attr):
@@ -291,7 +291,10 @@ def smart_update(self, mi):
                     setattr(self, attr, val)
 
         if mi.tags:
-            self.tags += mi.tags
+            if replace_tags:
+                self.tags = mi.tags
+            else:
+                self.tags += mi.tags
         self.tags = list(set(self.tags))
 
         if mi.author_sort_map:
@@ -314,6 +317,11 @@ def smart_update(self, mi):
         if len(other_comments.strip()) > len(my_comments.strip()):
             self.comments = other_comments
 
+        other_lang = getattr(mi, 'language', None)
+        if other_lang and other_lang.lower() != 'und':
+            self.language = other_lang
+
+
     def format_series_index(self):
         try:
             x = float(self.series_index)

src/calibre/ebooks/metadata/fetch.py

@@ -3,17 +3,18 @@
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import traceback, sys, textwrap, re
+import traceback, sys, textwrap, re, urllib2
 from threading import Thread
 
-from calibre import prints
+from calibre import prints, browser
 from calibre.utils.config import OptionParser
 from calibre.utils.logging import default_log
 from calibre.customize import Plugin
+from calibre.ebooks.metadata.library_thing import OPENLIBRARY
 
 metadata_config = None
 
-class MetadataSource(Plugin):
+class MetadataSource(Plugin): # {{{
 
     author = 'Kovid Goyal'
 
@@ -130,7 +131,9 @@ def save_settings(self, w):
     def customization_help(self):
         return 'This plugin can only be customized using the GUI'
 
-class GoogleBooks(MetadataSource):
+    # }}}
+
+class GoogleBooks(MetadataSource): # {{{
 
     name = 'Google Books'
     description = _('Downloads metadata from Google Books')
@@ -145,8 +148,9 @@ def fetch(self):
             self.exception = e
             self.tb = traceback.format_exc()
 
+    # }}}
 
-class ISBNDB(MetadataSource):
+class ISBNDB(MetadataSource): # {{{
 
     name = 'IsbnDB'
     description = _('Downloads metadata from isbndb.com')
@@ -181,7 +185,9 @@ def string_customization_help(self):
                 'and enter your access key below.')
         return '<p>'+ans%('<a href="http://www.isbndb.com">', '</a>')
 
-class Amazon(MetadataSource):
+    # }}}
+
+class Amazon(MetadataSource): # {{{
 
     name = 'Amazon'
     metadata_type = 'social'
@@ -198,37 +204,27 @@ def fetch(self):
             self.exception = e
             self.tb = traceback.format_exc()
 
-class LibraryThing(MetadataSource):
+    # }}}
+
+class LibraryThing(MetadataSource): # {{{
 
     name = 'LibraryThing'
     metadata_type = 'social'
-    description = _('Downloads series information from librarything.com')
+    description = _('Downloads series/tags/rating information from librarything.com')
 
     def fetch(self):
         if not self.isbn:
             return
-        from calibre import browser
-        from calibre.ebooks.metadata import MetaInformation
-        import json
-        br = browser()
+        from calibre.ebooks.metadata.library_thing import get_social_metadata
         try:
-            raw = br.open(
-                    'http://status.calibre-ebook.com/library_thing/metadata/'+self.isbn
-                    ).read()
-            data = json.loads(raw)
-            if not data:
-                return
-            if 'error' in data:
-                raise Exception(data['error'])
-            if 'series' in data and 'series_index' in data:
-                mi = MetaInformation(self.title, [])
-                mi.series = data['series']
-                mi.series_index = data['series_index']
-                self.results = mi
+            self.results = get_social_metadata(self.title, self.book_author,
+                    self.publisher, self.isbn)
         except Exception, e:
             self.exception = e
             self.tb = traceback.format_exc()
 
+    # }}}
+
 
 def result_index(source, result):
     if not result.isbn:
@@ -268,6 +264,31 @@ def join(self):
         for s in self.sources:
             s.join()
 
+def filter_metadata_results(item):
+    keywords = ["audio", "tape", "cassette", "abridged", "playaway"]
+    for keyword in keywords:
+        if item.publisher and keyword in item.publisher.lower():
+            return False
+    return True
+
+class HeadRequest(urllib2.Request):
+    def get_method(self):
+        return "HEAD"
+
+def do_cover_check(item):
+    opener = browser()
+    item.has_cover = False
+    try:
+        opener.open(HeadRequest(OPENLIBRARY%item.isbn), timeout=5)
+        item.has_cover = True
+    except:
+        pass # Cover not found
+
+def check_for_covers(items):
+    threads = [Thread(target=do_cover_check, args=(item,)) for item in items]
+    for t in threads: t.start()
+    for t in threads: t.join()
+
 def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
            verbose=0):
     assert not(title is None and author is None and publisher is None and \
@@ -285,10 +306,60 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
     for fetcher in fetchers[1:]:
         merge_results(results, fetcher.results)
 
-    results = sorted(results, cmp=lambda x, y : cmp(
-            (x.comments.strip() if x.comments else ''),
-            (y.comments.strip() if y.comments else '')
-                                                  ), reverse=True)
+    results = list(filter(filter_metadata_results, results))
+
+    check_for_covers(results)
+
+    words = ("the", "a", "an", "of", "and")
+    prefix_pat = re.compile(r'^(%s)\s+'%("|".join(words)))
+    trailing_paren_pat = re.compile(r'\(.*\)$')
+    whitespace_pat = re.compile(r'\s+')
+
+    def sort_func(x, y):
+
+        def cleanup_title(s):
+            s = s.strip().lower()
+            s = prefix_pat.sub(' ', s)
+            s = trailing_paren_pat.sub('', s)
+            s = whitespace_pat.sub(' ', s)
+            return s.strip()
+
+        t = cleanup_title(title)
+        x_title = cleanup_title(x.title)
+        y_title = cleanup_title(y.title)
+
+        # prefer titles that start with the search title
+        tx = cmp(t, x_title)
+        ty = cmp(t, y_title)
+        result = 0 if abs(tx) == abs(ty) else abs(tx) - abs(ty)
+
+        # then prefer titles that have a cover image
+        if result == 0:
+            result = -cmp(x.has_cover, y.has_cover)
+
+        # then prefer titles with the longest comment, with in 10%
+        if result == 0:
+            cx = len(x.comments.strip() if x.comments else '')
+            cy = len(y.comments.strip() if y.comments else '')
+            t = (cx + cy) / 20
+            result = cy - cx
+            if abs(result) < t:
+                result = 0
+
+        return result
+
+    results = sorted(results, cmp=sort_func)
+
+    # if for some reason there is no comment in the top selection, go looking for one
+    if len(results) > 1:
+        if not results[0].comments or len(results[0].comments) == 0:
+            for r in results[1:]:
+                if title.lower() == r.title[:len(title)].lower() and r.comments and len(r.comments):
+                    results[0].comments = r.comments
+                    break
+
+ #   for r in results:
+ #       print "{0:14.14} {1:30.30} {2:20.20} {3:6} {4}".format(r.isbn, r.title, r.publisher, len(r.comments if r.comments else ''), r.has_cover)
 
     return results, [(x.name, x.exception, x.tb) for x in fetchers]
 

src/calibre/ebooks/metadata/library_thing.py

@@ -6,10 +6,11 @@
 
 import sys, socket, os, re
 
-from calibre import browser as _browser
+from lxml import html
+
+from calibre import browser, prints
 from calibre.utils.config import OptionParser
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
-browser = None
 
 OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false'
 
@@ -22,31 +23,28 @@ class ISBNNotFound(LibraryThingError):
 class ServerBusy(LibraryThingError):
     pass
 
-def login(username, password, force=True):
-    global browser
-    if browser is not None and not force:
-        return
-    browser = _browser()
-    browser.open('http://www.librarything.com')
-    browser.select_form('signup')
-    browser['formusername'] = username
-    browser['formpassword'] = password
-    browser.submit()
+def login(br, username, password, force=True):
+    br.open('http://www.librarything.com')
+    br.select_form('signup')
+    br['formusername'] = username
+    br['formpassword'] = password
+    br.submit()
 
 
 def cover_from_isbn(isbn, timeout=5., username=None, password=None):
-    global browser
-    if browser is None:
-        browser = _browser()
     src = None
+    br = browser()
     try:
-        return browser.open(OPENLIBRARY%isbn, timeout=timeout).read(), 'jpg'
+        return br.open(OPENLIBRARY%isbn, timeout=timeout).read(), 'jpg'
     except:
         pass # Cover not found
     if username and password:
-        login(username, password, force=False)
+        try:
+            login(br, username, password, force=False)
+        except:
+            pass
     try:
-        src = browser.open('http://www.librarything.com/isbn/'+isbn,
+        src = br.open_novisit('http://www.librarything.com/isbn/'+isbn,
                 timeout=timeout).read().decode('utf-8', 'replace')
     except Exception, err:
         if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
@@ -63,7 +61,7 @@ def cover_from_isbn(isbn, timeout=5., username=None, password=None):
         if url is None:
             raise LibraryThingError(_('LibraryThing.com server error. Try again later.'))
         url = re.sub(r'_S[XY]\d+', '', url['src'])
-        cover_data = browser.open(url).read()
+        cover_data = br.open_novisit(url).read()
         return cover_data, url.rpartition('.')[-1]
 
 def option_parser():
@@ -71,7 +69,7 @@ def option_parser():
 _('''
 %prog [options] ISBN
 
-Fetch a cover image for the book identified by ISBN from LibraryThing.com
+Fetch a cover image/social metadata for the book identified by ISBN from LibraryThing.com
 '''))
     parser.add_option('-u', '--username', default=None,
                       help='Username for LibraryThing.com')
@@ -79,6 +77,61 @@ def option_parser():
                       help='Password for LibraryThing.com')
     return parser
 
+def get_social_metadata(title, authors, publisher, isbn, username=None,
+        password=None):
+    from calibre.ebooks.metadata import MetaInformation
+    mi = MetaInformation(title, authors)
+    if isbn:
+        br = browser()
+        if username and password:
+            try:
+                login(br, username, password, force=False)
+            except:
+                pass
+
+        raw = br.open_novisit('http://www.librarything.com/isbn/'
+                    +isbn).read()
+        if not raw:
+            return mi
+        root = html.fromstring(raw)
+        h1 = root.xpath('//div[@class="headsummary"]/h1')
+        if h1 and not mi.title:
+            mi.title = html.tostring(h1[0], method='text', encoding=unicode)
+        h2 = root.xpath('//div[@class="headsummary"]/h2/a')
+        if h2 and not mi.authors:
+            mi.authors = [html.tostring(x, method='text', encoding=unicode) for
+                    x in h2]
+        h3 = root.xpath('//div[@class="headsummary"]/h3/a')
+        if h3:
+            match = None
+            for h in h3:
+               series = html.tostring(h, method='text', encoding=unicode)
+               match = re.search(r'(.+) \((.+)\)', series)
+               if match is not None:
+                   break
+            if match is not None:
+                mi.series = match.group(1).strip()
+                match = re.search(r'[0-9.]+', match.group(2))
+                si = 1.0
+                if match is not None:
+                    si = float(match.group())
+                mi.series_index = si
+        tags = root.xpath('//div[@class="tags"]/span[@class="tag"]/a')
+        if tags:
+            mi.tags = [html.tostring(x, method='text', encoding=unicode) for x
+                    in tags]
+        span = root.xpath(
+                '//table[@class="wsltable"]/tr[@class="wslcontent"]/td[4]//span')
+        if span:
+            raw = html.tostring(span[0], method='text', encoding=unicode)
+            match = re.search(r'([0-9.]+)', raw)
+            if match is not None:
+                rating = float(match.group())
+                if rating > 0 and rating <= 5:
+                    mi.rating = rating
+    return mi
+
+
 def main(args=sys.argv):
     parser = option_parser()
     opts, args = parser.parse_args(args)
@@ -86,6 +139,8 @@ def main(args=sys.argv):
         parser.print_help()
         return 1
     isbn = args[1]
+    mi = get_social_metadata('', [], '', isbn)
+    prints(mi)
     cover_data, ext = cover_from_isbn(isbn, username=opts.username,
             password=opts.password)
     if not ext:

src/calibre/ebooks/metadata/opf2.py

@@ -18,7 +18,7 @@
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.metadata import MetaInformation, string_to_authors
 from calibre.utils.date import parse_date, isoformat
-
+from calibre.utils.localization import get_lang
 
 class Resource(object):
     '''
@@ -1069,7 +1069,7 @@ def CAL_ELEM(name, content):
             dc_attrs={'id':__appname__+'_id'}))
         if getattr(self, 'pubdate', None) is not None:
             a(DC_ELEM('date', self.pubdate.isoformat()))
-        a(DC_ELEM('language', self.language if self.language else 'UND'))
+        a(DC_ELEM('language', self.language if self.language else get_lang()))
         if self.comments:
             a(DC_ELEM('description', self.comments))
         if self.publisher:
@@ -1184,7 +1184,6 @@ def factory(tag, text=None, sort=None, role=None, scheme=None, name=None,
     factory(DC('contributor'), mi.book_producer, __appname__, 'bkp')
     if hasattr(mi.pubdate, 'isoformat'):
         factory(DC('date'), isoformat(mi.pubdate))
-    factory(DC('language'), mi.language)
     if mi.category:
         factory(DC('type'), mi.category)
     if mi.comments:
@@ -1195,6 +1194,7 @@ def factory(tag, text=None, sort=None, role=None, scheme=None, name=None,
         factory(DC('identifier'), mi.isbn, scheme='ISBN')
     if mi.rights:
         factory(DC('rights'), mi.rights)
+    factory(DC('language'), mi.language if mi.language and mi.language.lower() != 'und' else get_lang())
     if mi.tags:
         for tag in mi.tags:
             factory(DC('subject'), tag)

src/calibre/ebooks/oeb/base.py

@@ -17,6 +17,7 @@
 
 from lxml import etree, html
 from cssutils import CSSParser
+from cssutils.css import CSSRule
 
 import calibre
 from calibre.constants import filesystem_encoding
@@ -762,6 +763,7 @@ def __init__(self, oeb, id, href, media_type,
             self.href = self.path = urlnormalize(href)
             self.media_type = media_type
             self.fallback = fallback
+            self.override_css_fetch = None
             self.spine_position = None
             self.linear = True
             if loader is None and data is None:
@@ -982,15 +984,40 @@ def _parse_txt(self, data):
 
 
         def _parse_css(self, data):
+
+            def get_style_rules_from_import(import_rule):
+                ans = []
+                if not import_rule.styleSheet:
+                    return ans
+                rules = import_rule.styleSheet.cssRules
+                for rule in rules:
+                    if rule.type == CSSRule.IMPORT_RULE:
+                        ans.extend(get_style_rules_from_import(rule))
+                    elif rule.type in (CSSRule.FONT_FACE_RULE,
+                            CSSRule.STYLE_RULE):
+                        ans.append(rule)
+                return ans
+
             self.oeb.log.debug('Parsing', self.href, '...')
             data = self.oeb.decode(data)
-            data = self.oeb.css_preprocessor(data)
-            data = XHTML_CSS_NAMESPACE + data
+            data = self.oeb.css_preprocessor(data, add_namespace=True)
             parser = CSSParser(loglevel=logging.WARNING,
-                               fetcher=self._fetch_css,
+                               fetcher=self.override_css_fetch or self._fetch_css,
                                log=_css_logger)
             data = parser.parseString(data, href=self.href)
             data.namespaces['h'] = XHTML_NS
+            import_rules = list(data.cssRules.rulesOfType(CSSRule.IMPORT_RULE))
+            rules_to_append = []
+            insert_index = None
+            for r in data.cssRules.rulesOfType(CSSRule.STYLE_RULE):
+                insert_index = data.cssRules.index(r)
+                break
+            for rule in import_rules:
+                rules_to_append.extend(get_style_rules_from_import(rule))
+            for r in reversed(rules_to_append):
+                data.insertRule(r, index=insert_index)
+            for rule in import_rules:
+                data.deleteRule(rule)
             return data
 
         def _fetch_css(self, path):

src/calibre/ebooks/oeb/iterator.py

@@ -139,11 +139,18 @@ def find_embedded_fonts(self):
                     if id != -1:
                         families = [unicode(f) for f in QFontDatabase.applicationFontFamilies(id)]
                         if family:
-                            family = family.group(1).strip().replace('"', '')
-                            bad_map[family] = families[0]
-                            if family not in families:
+                            family = family.group(1)
+                            specified_families = [x.strip().replace('"',
+                                '').replace("'", '') for x in family.split(',')]
+                            aliasing_ok = False
+                            for f in specified_families:
+                                bad_map[f] = families[0]
+                                if not aliasing_ok and f in families:
+                                    aliasing_ok = True
+
+                            if not aliasing_ok:
                                 prints('WARNING: Family aliasing not fully supported.')
-                                prints('\tDeclared family: %s not in actual families: %s'
+                                prints('\tDeclared family: %r not in actual families: %r'
                                         % (family, families))
                             else:
                                 prints('Loaded embedded font:', repr(family))

src/calibre/ebooks/oeb/stylizer.py

@@ -126,6 +126,13 @@ def __init__(self, tree, path, oeb, opts, profile=PROFILES['PRS505'],
             head = head[0]
         else:
             head = []
+
+        # Add cssutils parsing profiles from output_profile
+        for profile in self.opts.output_profile.extra_css_modules:
+            cssutils.profile.addProfile(profile['name'],
+                                        profile['props'],
+                                        profile['macros'])
+
         parser = cssutils.CSSParser(fetcher=self._fetch_css_file,
                 log=logging.getLogger('calibre.css'))
         self.font_face_rules = []

src/calibre/gui2/actions.py

@@ -21,6 +21,7 @@
 from calibre.gui2.widgets import IMAGE_EXTENSIONS
 from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
 from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
+from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebook, \
     fetch_scheduled_recipe, generate_catalog
 from calibre.constants import preferred_encoding, filesystem_encoding, \
@@ -176,7 +177,8 @@ def generate_annotation_html(self, bookmark):
 
             def mark_book_as_read(self,id):
                 read_tag = gprefs.get('catalog_epub_mobi_read_tag')
-                self.db.set_tags(id, [read_tag], append=True)
+                if read_tag:
+                    self.db.set_tags(id, [read_tag], append=True)
 
             def canceled(self):
                 self.pd.hide()
@@ -830,6 +832,23 @@ def merge_metadata(self, dest_id, src_ids):
         db.set_metadata(dest_id, dest_mi, ignore_errors=False)
         # }}}
 
+    def edit_device_collections(self, view, oncard=None):
+        model = view.model()
+        result = model.get_collections_with_ids()
+        compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        d = TagListEditor(self, tag_to_match=None, data=result, compare=compare)
+        d.exec_()
+        if d.result() == d.Accepted:
+            to_rename = d.to_rename # dict of new text to old ids
+            to_delete = d.to_delete # list of ids
+            for text in to_rename:
+                for old_id in to_rename[text]:
+                    model.rename_collection(old_id, new_name=unicode(text))
+            for item in to_delete:
+                model.delete_collection_using_id(item)
+            self.upload_collections(model.db, view=view, oncard=oncard)
+            view.reset()
+
     # }}}
 
 class SaveToDiskAction(object): # {{{

src/calibre/gui2/convert/txt_input.ui

@@ -43,6 +43,9 @@
      <property name="wordWrap">
       <bool>true</bool>
      </property>
+     <property name="openExternalLinks">
+      <bool>true</bool>
+     </property>
     </widget>
    </item>
    <item row="4" column="0">

src/calibre/gui2/custom_column_widgets.py

@@ -5,7 +5,7 @@
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys
+import re, sys
 from functools import partial
 
 from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \
@@ -162,7 +162,6 @@ def getter(self):
             val = qt_to_dt(val)
         return val
 
-
 class Comments(Base):
 
     def setup_ui(self, parent):
@@ -199,11 +198,7 @@ def setup_ui(self, parent):
             w = EnComboBox(parent)
             w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
             w.setMinimumContentsLength(25)
-
-
-
-        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent),
-                w]
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
 
     def initialize(self, book_id):
         val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
@@ -222,7 +217,6 @@ def initialize(self, book_id):
             if idx is not None:
                 self.widgets[1].setCurrentIndex(idx)
 
-
     def setter(self, val):
         if self.col_metadata['is_multiple']:
             if not val:
@@ -241,6 +235,58 @@ def getter(self):
             val = None
         return val
 
+class Series(Base):
+
+    def setup_ui(self, parent):
+        values = self.all_values = list(self.db.all_custom(num=self.col_id))
+        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        w = EnComboBox(parent)
+        w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
+        w.setMinimumContentsLength(25)
+        self.name_widget = w
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
+
+        self.widgets.append(QLabel('&'+self.col_metadata['name']+_(' index:'), parent))
+        w = QDoubleSpinBox(parent)
+        w.setRange(-100., float(sys.maxint))
+        w.setDecimals(2)
+        w.setSpecialValueText(_('Undefined'))
+        w.setSingleStep(1)
+        self.idx_widget=w
+        self.widgets.append(w)
+
+    def initialize(self, book_id):
+        val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
+        s_index = self.db.get_custom_extra(book_id, num=self.col_id, index_is_id=True)
+        if s_index is None:
+            s_index = 0.0
+        self.idx_widget.setValue(s_index)
+        self.initial_index = s_index
+        self.initial_val = val
+        val = self.normalize_db_val(val)
+        idx = None
+        for i, c in enumerate(self.all_values):
+            if c == val:
+                idx = i
+            self.name_widget.addItem(c)
+        self.name_widget.setEditText('')
+        if idx is not None:
+            self.widgets[1].setCurrentIndex(idx)
+
+    def commit(self, book_id, notify=False):
+        val = unicode(self.name_widget.currentText()).strip()
+        val = self.normalize_ui_val(val)
+        s_index = self.idx_widget.value()
+        if val != self.initial_val or s_index != self.initial_index:
+            if s_index == 0.0:
+                if tweaks['series_index_auto_increment'] == 'next':
+                    s_index = self.db.get_next_cc_series_num_for(val,
+                                                             num=self.col_id)
+                else:
+                    s_index = None
+            self.db.set_custom(book_id, val, extra=s_index,
+                               num=self.col_id, notify=notify)
+
 widgets = {
         'bool' : Bool,
         'rating' : Rating,
@@ -249,6 +295,7 @@ def getter(self):
         'datetime': DateTime,
         'text' : Text,
         'comments': Comments,
+        'series': Series,
 }
 
 def field_sort(y, z, x=None):
@@ -257,35 +304,63 @@ def field_sort(y, z, x=None):
     n2 = 'zzzzz' if m2['datatype'] == 'comments' else m2['name']
     return cmp(n1.lower(), n2.lower())
 
-def populate_single_metadata_page(left, right, db, book_id, parent=None):
+def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, parent=None):
+    def widget_factory(type, col):
+        if bulk:
+            w = bulk_widgets[type](db, col, parent)
+        else:
+            w = widgets[type](db, col, parent)
+        w.initialize(book_id)
+        return w
     x = db.custom_column_num_map
     cols = list(x)
     cols.sort(cmp=partial(field_sort, x=x))
+    count_non_comment = len([c for c in cols if x[c]['datatype'] != 'comments'])
+
+    layout.setColumnStretch(1, 10)
+    if two_column:
+        turnover_point = (count_non_comment+1)/2
+        layout.setColumnStretch(3, 10)
+    else:
+        # Avoid problems with multi-line widgets
+        turnover_point = count_non_comment + 1000
     ans = []
-    for i, col in enumerate(cols):
-        w = widgets[x[col]['datatype']](db, col, parent)
+    column = row = 0
+    for col in cols:
+        dt = x[col]['datatype']
+        if dt == 'comments':
+            continue
+        w = widget_factory(dt, col)
         ans.append(w)
-        w.initialize(book_id)
-        layout = left if i%2 == 0 else right
-        row = layout.rowCount()
-        if len(w.widgets) == 1:
-            layout.addWidget(w.widgets[0], row, 0, 1, -1)
-        else:
-            w.widgets[0].setBuddy(w.widgets[1])
-            for c, widget in enumerate(w.widgets):
-                layout.addWidget(widget, row, c)
+        for c in range(0, len(w.widgets), 2):
+            w.widgets[c].setBuddy(w.widgets[c+1])
+            layout.addWidget(w.widgets[c], row, column)
+            layout.addWidget(w.widgets[c+1], row, column+1)
+            row += 1
+        if row >= turnover_point:
+            column += 2
+            turnover_point = count_non_comment + 1000
+            row = 0
+    if not bulk: # Add the comments fields
+        column = 0
+        for col in cols:
+            dt = x[col]['datatype']
+            if dt != 'comments':
+                continue
+            w = widget_factory(dt, col)
+            ans.append(w)
+            layout.addWidget(w.widgets[0], row, column, 1, 2)
+            if two_column and column == 0:
+                column = 2
+                continue
+            column = 0
+            row += 1
     items = []
     if len(ans) > 0:
         items.append(QSpacerItem(10, 10, QSizePolicy.Minimum,
             QSizePolicy.Expanding))
-        left.addItem(items[-1], left.rowCount(), 0, 1, 1)
-        left.setRowStretch(left.rowCount()-1, 100)
-    if len(ans) > 1:
-         items.append(QSpacerItem(10, 100, QSizePolicy.Minimum,
-             QSizePolicy.Expanding))
-         right.addItem(items[-1], left.rowCount(), 0, 1, 1)
-         right.setRowStretch(right.rowCount()-1, 100)
-
+        layout.addItem(items[-1], layout.rowCount(), 0, 1, 1)
+        layout.setRowStretch(layout.rowCount()-1, 100)
     return ans, items
 
 class BulkBase(Base):
@@ -342,6 +417,47 @@ class BulkRating(BulkBase, Rating):
 class BulkDateTime(BulkBase, DateTime):
     pass
 
+class BulkSeries(BulkBase):
+    def setup_ui(self, parent):
+        values = self.all_values = list(self.db.all_custom(num=self.col_id))
+        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        w = EnComboBox(parent)
+        w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
+        w.setMinimumContentsLength(25)
+        self.name_widget = w
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
+
+        self.widgets.append(QLabel(_('Automatically number books in this series'), parent))
+        self.idx_widget=QCheckBox(parent)
+        self.widgets.append(self.idx_widget)
+
+    def initialize(self, book_id):
+        self.idx_widget.setChecked(False)
+        for c in self.all_values:
+            self.name_widget.addItem(c)
+        self.name_widget.setEditText('')
+
+    def commit(self, book_ids, notify=False):
+        val = unicode(self.name_widget.currentText()).strip()
+        val = self.normalize_ui_val(val)
+        update_indices = self.idx_widget.checkState()
+        if val != '':
+            for book_id in book_ids:
+                if update_indices:
+                    if tweaks['series_index_auto_increment'] == 'next':
+                        s_index = self.db.get_next_cc_series_num_for\
+                                                        (val, num=self.col_id)
+                    else:
+                        s_index = 1.0
+                else:
+                    s_index = self.db.get_custom_extra(book_id, num=self.col_id,
+                                                       index_is_id=True)
+                self.db.set_custom(book_id, val, extra=s_index,
+                                   num=self.col_id, notify=notify)
+
+    def process_each_book(self):
+        return True
+
 class RemoveTags(QWidget):
 
     def __init__(self, parent, values):
@@ -431,35 +547,5 @@ def getter(self, original_value = None):
         'float': BulkFloat,
         'datetime': BulkDateTime,
         'text' : BulkText,
-}
-
-def populate_bulk_metadata_page(layout, db, book_ids, parent=None):
-    x = db.custom_column_num_map
-    cols = list(x)
-    cols.sort(cmp=partial(field_sort, x=x))
-    ans = []
-    for i, col in enumerate(cols):
-        dt = x[col]['datatype']
-        if dt == 'comments':
-            continue
-        w = bulk_widgets[dt](db, col, parent)
-        ans.append(w)
-        w.initialize(book_ids)
-        row = layout.rowCount()
-        if len(w.widgets) == 1:
-            layout.addWidget(w.widgets[0], row, 0, 1, -1)
-        else:
-            for c in range(0, len(w.widgets), 2):
-                w.widgets[c].setBuddy(w.widgets[c+1])
-                layout.addWidget(w.widgets[c], row, 0)
-                layout.addWidget(w.widgets[c+1], row, 1)
-                row += 1
-    items = []
-    if len(ans) > 0:
-        items.append(QSpacerItem(10, 10, QSizePolicy.Minimum,
-            QSizePolicy.Expanding))
-        layout.addItem(items[-1], layout.rowCount(), 0, 1, 1)
-        layout.setRowStretch(layout.rowCount()-1, 100)
-
-    return ans, items
-
+        'series': BulkSeries,
+}

src/calibre/gui2/device.py

@@ -10,7 +10,7 @@
 from binascii import unhexlify
 
 from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, QPixmap, \
-                     Qt, pyqtSignal, QColor, QPainter
+                     Qt, pyqtSignal, QColor, QPainter, QDialog
 from PyQt4.QtSvg import QSvgRenderer
 
 from calibre.customize.ui import available_input_formats, available_output_formats, \
@@ -294,6 +294,11 @@ def sync_booklists(self, done, booklists):
         return self.create_job(self._sync_booklists, done, args=[booklists],
                         description=_('Send metadata to device'))
 
+    def upload_collections(self, done, booklist, on_card):
+        return self.create_job(booklist.rebuild_collections, done,
+                               args=[booklist, on_card],
+                        description=_('Send collections to device'))
+
     def _upload_books(self, files, names, on_card=None, metadata=None):
         '''Upload books to device: '''
         return self.device.upload_books(files, names, on_card,
@@ -814,7 +819,8 @@ def dispatch_sync_event(self, dest, delete, specific):
         if specific:
             d = ChooseFormatDialog(self, _('Choose format to send to device'),
                                 self.device_manager.device.settings().format_map)
-            d.exec_()
+            if d.exec_() != QDialog.Accepted:
+                return
             if d.format():
                 fmt = d.format().lower()
         dest, sub_dest = dest.split(':')
@@ -1227,6 +1233,19 @@ def metadata_synced(self, job):
             return
         cp, fs = job.result
         self.location_view.model().update_devices(cp, fs)
+        # reset the views so that up-to-date info is shown. These need to be
+        # here because the sony driver updates collections in sync_booklists
+        self.memory_view.reset()
+        self.card_a_view.reset()
+        self.card_b_view.reset()
+
+    def _upload_collections(self, job):
+        if job.failed:
+            self.device_job_exception(job)
+
+    def upload_collections(self, booklist, view=None, oncard=None):
+        return self.device_manager.upload_collections(self._upload_collections,
+                                                       booklist, oncard)
 
     def upload_books(self, files, names, metadata, on_card=None, memory=None):
         '''

src/calibre/gui2/dialogs/choose_format.ui

@@ -39,7 +39,7 @@
       <enum>Qt::Horizontal</enum>
      </property>
      <property name="standardButtons" >
-      <set>QDialogButtonBox::Ok</set>
+      <set>QDialogButtonBox::Ok|QDialogButtonBox::Cancel</set>
      </property>
     </widget>
    </item>

src/calibre/gui2/dialogs/config/add_save.py

@@ -45,6 +45,7 @@ def __init__(self, parent=None):
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)
         self.opt_swap_author_names.setChecked(prefs['swap_author_names'])
         self.opt_add_formats_to_existing.setChecked(prefs['add_formats_to_existing'])
+        self.preserve_user_collections.setChecked(prefs['preserve_user_collections'])
         help = '\n'.join(textwrap.wrap(c.get_option('template').help, 75))
         self.save_template.initialize('save_to_disk', opts.template, help)
         self.send_template.initialize('send_to_device', opts.send_template, help)
@@ -71,6 +72,7 @@ def save_settings(self):
         prefs['filename_pattern'] = pattern
         prefs['swap_author_names'] = bool(self.opt_swap_author_names.isChecked())
         prefs['add_formats_to_existing'] = bool(self.opt_add_formats_to_existing.isChecked())
+        prefs['preserve_user_collections'] = bool(self.preserve_user_collections.isChecked())
 
         return True
 

src/calibre/gui2/dialogs/config/add_save.ui

@@ -51,7 +51,7 @@
     <item row="2" column="0" colspan="2">
      <widget class="QCheckBox" name="opt_add_formats_to_existing">
       <property name="toolTip">
-       <string>If an existing book with a similar title and author is found that does not have the format being added, the format is added 
+       <string>If an existing book with a similar title and author is found that does not have the format being added, the format is added
 to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored.
 
 Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc. Author match is exact.</string>
@@ -179,7 +179,31 @@ Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;,
    </attribute>
    <layout class="QVBoxLayout" name="verticalLayout_2">
     <item>
-     <widget class="QLabel" name="label_4">
+     <widget class="QCheckBox" name="preserve_user_collections">
+      <property name="text">
+       <string>Preserve device collections.</string>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_41">
+      <property name="text">
+       <string>If checked, collections will not be deleted even if a book with changed metadata is resent and the collection is not in the book's metadata. In addition, editing collections in the device view will be enabled. If unchecked, collections will be always reflect only the metadata in the calibre library.</string>
+      </property>
+      <property name="wordWrap">
+       <bool>true</bool>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_42">
+      <property name="text">
+       <string>  </string>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_43">
       <property name="text">
        <string>Here you can control how calibre will save your books when you click the Send to Device button. This setting can be overriden for individual devices by customizing the device interface plugins in Preferences-&gt;Plugins</string>
       </property>

src/calibre/gui2/dialogs/config/create_custom_column.py

@@ -24,16 +24,19 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                     2:{'datatype':'comments',
                         'text':_('Long text, like comments, not shown in the tag browser'),
                         'is_multiple':False},
-                    3:{'datatype':'datetime',
+                    3:{'datatype':'series',
+                        'text':_('Text column for keeping series-like information'),
+                        'is_multiple':False},
+                    4:{'datatype':'datetime',
                         'text':_('Date'), 'is_multiple':False},
-                    4:{'datatype':'float',
+                    5:{'datatype':'float',
                         'text':_('Floating point numbers'), 'is_multiple':False},
-                    5:{'datatype':'int',
+                    6:{'datatype':'int',
                         'text':_('Integers'), 'is_multiple':False},
-                    6:{'datatype':'rating',
+                    7:{'datatype':'rating',
                         'text':_('Ratings, shown with stars'),
                         'is_multiple':False},
-                    7:{'datatype':'bool',
+                    8:{'datatype':'bool',
                         'text':_('Yes/No'), 'is_multiple':False},
                 }
 

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -10,7 +10,7 @@
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.ebooks.metadata import string_to_authors, \
     authors_to_string
-from calibre.gui2.custom_column_widgets import populate_bulk_metadata_page
+from calibre.gui2.custom_column_widgets import populate_metadata_page
 
 class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
@@ -44,15 +44,14 @@ def __init__(self, window, rows, db):
             self.central_widget.tabBar().setVisible(False)
         else:
             self.create_custom_column_editors()
-
         self.exec_()
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
         layout = QGridLayout()
-
-        self.custom_column_widgets, self.__cc_spacers = populate_bulk_metadata_page(
-                layout, self.db, self.ids, w)
+        self.custom_column_widgets, self.__cc_spacers = \
+            populate_metadata_page(layout, self.db, self.ids, parent=w,
+                                   two_column=False, bulk=True)
         w.setLayout(layout)
         self.__custom_col_layouts = [layout]
         ans = self.custom_column_widgets

src/calibre/gui2/dialogs/metadata_single.py

@@ -32,7 +32,7 @@
 from calibre.utils.date import qt_to_dt
 from calibre.customize.ui import run_plugins_on_import, get_isbndb_key
 from calibre.gui2.dialogs.config.social import SocialMetadata
-from calibre.gui2.custom_column_widgets import populate_single_metadata_page
+from calibre.gui2.custom_column_widgets import populate_metadata_page
 
 class CoverFetcher(QThread):
 
@@ -420,23 +420,19 @@ def __init__(self, window, row, db, accepted_callback=None, cancel_all=False):
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
-        top_layout = QHBoxLayout()
-        top_layout.setSpacing(20)
-        left_layout = QGridLayout()
-        right_layout = QGridLayout()
-        top_layout.addLayout(left_layout)
-
-        self.custom_column_widgets, self.__cc_spacers = populate_single_metadata_page(
-                left_layout, right_layout, self.db, self.id, w)
-        top_layout.addLayout(right_layout)
-        sip.delete(w.layout())
-        w.setLayout(top_layout)
-        self.__custom_col_layouts = [top_layout, left_layout, right_layout]
+        layout = w.layout()
+        self.custom_column_widgets, self.__cc_spacers = \
+                    populate_metadata_page(layout, self.db, self.id,
+                                           parent=w, bulk=False, two_column=True)
+        self.__custom_col_layouts = [layout]
         ans = self.custom_column_widgets
         for i in range(len(ans)-1):
-            w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[-1])
-
-
+            if len(ans[i+1].widgets) == 2:
+                w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[1])
+            else:
+                w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[0])
+            for c in range(2, len(ans[i].widgets), 2):
+                w.setTabOrder(ans[i].widgets[c-1], ans[i].widgets[c+1])
 
     def validate_isbn(self, isbn):
         isbn = unicode(isbn).strip()

src/calibre/gui2/dialogs/tag_list_editor.py

@@ -1,54 +1,64 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-from functools import partial
 from PyQt4.QtCore import SIGNAL, Qt
 from PyQt4.QtGui import QDialog, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
 from calibre.gui2 import question_dialog, error_dialog
-from calibre.ebooks.metadata import title_sort
+
+class ListWidgetItem(QListWidgetItem):
+
+    def __init__(self, txt):
+        QListWidgetItem.__init__(self, txt)
+        self.old_value = txt
+        self.cur_value = txt
+
+    def data(self, role):
+        if role == Qt.DisplayRole:
+            if self.old_value != self.cur_value:
+                return _('%s (was %s)'%(self.cur_value, self.old_value))
+            else:
+                return self.cur_value
+        elif role == Qt.EditRole:
+            return self.cur_value
+        else:
+            return QListWidgetItem.data(self, role)
+
+    def setData(self, role, data):
+        if role == Qt.EditRole:
+            self.cur_value = data.toString()
+        QListWidgetItem.setData(self, role, data)
+
+    def text(self):
+        return self.cur_value
+
+    def setText(self, txt):
+        self.cur_value = txt
+        QListWidgetItem.setText(txt)
 
 class TagListEditor(QDialog, Ui_TagListEditor):
 
-    def __init__(self, window, db, tag_to_match, category):
+    def __init__(self, window, tag_to_match, data, compare):
         QDialog.__init__(self, window)
         Ui_TagListEditor.__init__(self)
         self.setupUi(self)
 
         self.to_rename = {}
         self.to_delete = []
-        self.db = db
         self.all_tags = {}
-        self.category = category
-        if category == 'tags':
-            result = db.get_tags_with_ids()
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
-        elif category == 'series':
-            result = db.get_series_with_ids()
-            compare = (lambda x,y:cmp(title_sort(x).lower(), title_sort(y).lower()))
-        elif category == 'publisher':
-            result = db.get_publishers_with_ids()
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
-        else: # should be a custom field
-            self.cc_label = None
-            if category in db.field_metadata:
-                self.cc_label = db.field_metadata[category]['label']
-                result = self.db.get_custom_items_with_ids(label=self.cc_label)
-            else:
-                result = []
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
 
-        for k,v in result:
+        for k,v in data:
             self.all_tags[v] = k
         for tag in sorted(self.all_tags.keys(), cmp=compare):
-            item = QListWidgetItem(tag)
+            item = ListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
             self.available_tags.addItem(item)
 
-        items = self.available_tags.findItems(tag_to_match, Qt.MatchExactly)
-        if len(items) == 1:
-            self.available_tags.setCurrentItem(items[0])
+        if tag_to_match is not None:
+            items = self.available_tags.findItems(tag_to_match, Qt.MatchExactly)
+            if len(items) == 1:
+                self.available_tags.setCurrentItem(items[0])
 
         self.connect(self.delete_button,  SIGNAL('clicked()'), self.delete_tags)
         self.connect(self.rename_button,  SIGNAL('clicked()'), self.rename_tag)
@@ -62,13 +72,11 @@ def finish_editing(self, item):
                 item.setText(self.item_before_editing.text())
                 return
         if item.text() != self.item_before_editing.text():
-            if item.text() in self.all_tags.keys() or item.text() in self.to_rename.keys():
-                error_dialog(self, _('Item already used'),
-                             _('The item %s is already used.')%(item.text())).exec_()
-                item.setText(self.item_before_editing.text())
-                return
             (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
-            self.to_rename[item.text()] = id
+            if item.text() not in self.to_rename:
+                self.to_rename[item.text()] = [id]
+            else:
+                self.to_rename[item.text()].append(id)
 
     def rename_tag(self):
         item = self.available_tags.currentItem()
@@ -99,30 +107,3 @@ def delete_tags(self, item=None):
             self.to_delete.append(id)
             self.available_tags.takeItem(self.available_tags.row(item))
 
-    def accept(self):
-        rename_func = None
-        if self.category == 'tags':
-            rename_func = self.db.rename_tag
-            delete_func = self.db.delete_tag_using_id
-        elif self.category == 'series':
-            rename_func = self.db.rename_series
-            delete_func = self.db.delete_series_using_id
-        elif self.category == 'publisher':
-            rename_func = self.db.rename_publisher
-            delete_func = self.db.delete_publisher_using_id
-        else:
-            rename_func = partial(self.db.rename_custom_item, label=self.cc_label)
-            delete_func = partial(self.db.delete_custom_item_using_id, label=self.cc_label)
-
-        work_done = False
-        if rename_func:
-            for text in self.to_rename:
-                work_done = True
-                rename_func(id=self.to_rename[text], new_name=unicode(text))
-            for item in self.to_delete:
-                work_done = True
-                delete_func(item)
-        if not work_done:
-            QDialog.reject(self)
-        else:
-            QDialog.accept(self)

src/calibre/gui2/init.py

@@ -226,17 +226,30 @@ def __init__(self, db):
                                         self.action_show_book_details,
                                         self.action_del,
                                         add_to_library = None,
+                                        edit_device_collections=None,
                                         similar_menu=similar_menu)
         add_to_library = (_('Add books to library'), self.add_books_from_device)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard=None))
         self.memory_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard='carda'))
         self.card_a_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard='cardb'))
         self.card_b_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
 
         self.library_view.files_dropped.connect(self.files_dropped, type=Qt.QueuedConnection)
         for func, args in [
@@ -249,9 +262,14 @@ def __init__(self, db):
                 getattr(view, func)(*args)
 
         self.memory_view.connect_dirtied_signal(self.upload_booklists)
+        self.memory_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard=None)
         self.card_a_view.connect_dirtied_signal(self.upload_booklists)
+        self.card_a_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard='carda')
         self.card_b_view.connect_dirtied_signal(self.upload_booklists)
-
+        self.card_b_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard='cardb')
         self.book_on_device(None, reset=True)
         db.set_book_on_device_func(self.book_on_device)
         self.library_view.set_database(db)

src/calibre/gui2/library/models.py

@@ -16,11 +16,12 @@
 from calibre.utils.pyparsing import ParseException
 from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH
+from calibre.library.cli import parse_series_string
 from calibre import strftime, isbytestring, prepare_string_for_xml
 from calibre.constants import filesystem_encoding
 from calibre.gui2.library import DEFAULT_SORT
@@ -520,7 +521,7 @@ def tags(r, idx=-1):
                 return QVariant(', '.join(sorted(tags.split(','))))
             return None
 
-        def series(r, idx=-1, siix=-1):
+        def series_type(r, idx=-1, siix=-1):
             series = self.db.data[r][idx]
             if series:
                 idx = fmt_sidx(self.db.data[r][siix])
@@ -591,7 +592,7 @@ def number_type(r, idx=-1):
                                 idx=self.db.field_metadata['publisher']['rec_index'], mult=False),
                    'tags'     : functools.partial(tags,
                                 idx=self.db.field_metadata['tags']['rec_index']),
-                   'series'   : functools.partial(series,
+                   'series'   : functools.partial(series_type,
                                 idx=self.db.field_metadata['series']['rec_index'],
                                 siix=self.db.field_metadata['series_index']['rec_index']),
                    'ondevice' : functools.partial(text_type,
@@ -620,6 +621,9 @@ def number_type(r, idx=-1):
                                             bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes')
             elif datatype == 'rating':
                 self.dc[col] = functools.partial(rating_type, idx=idx)
+            elif datatype == 'series':
+                self.dc[col] = functools.partial(series_type, idx=idx,
+                    siix=self.db.field_metadata.cc_series_index_column_for(col))
             else:
                 print 'What type is this?', col, datatype
         # build a index column to data converter map, to remove the string lookup in the data loop
@@ -681,6 +685,8 @@ def flags(self, index):
 
     def set_custom_column_data(self, row, colhead, value):
         typ = self.custom_columns[colhead]['datatype']
+        label=self.db.field_metadata.key_to_label(colhead)
+        s_index = None
         if typ in ('text', 'comments'):
             val = unicode(value.toString()).strip()
             val = val if val else None
@@ -702,9 +708,10 @@ def set_custom_column_data(self, row, colhead, value):
                 if not val.isValid():
                     return False
                 val = qt_to_dt(val, as_utc=False)
-        self.db.set_custom(self.db.id(row), val,
-                           label=self.db.field_metadata.key_to_label(colhead),
-                           num=None, append=False, notify=True)
+        elif typ == 'series':
+            val, s_index = parse_series_string(self.db, label, value.toString())
+        self.db.set_custom(self.db.id(row), val, extra=s_index,
+                           label=label, num=None, append=False, notify=True)
         return True
 
     def setData(self, index, value, role):
@@ -850,6 +857,7 @@ def get_matches(self, location, query):
 class DeviceBooksModel(BooksModel): # {{{
 
     booklist_dirtied = pyqtSignal()
+    upload_collections = pyqtSignal(object)
 
     def __init__(self, parent):
         BooksModel.__init__(self, parent)
@@ -920,11 +928,12 @@ def flags(self, index):
         if index.isValid() and self.editable:
             cname = self.column_map[index.column()]
             if cname in ('title', 'authors') or \
-                    (cname == 'collections' and self.db.supports_collections()):
+                    (cname == 'collections' and \
+                     self.db.supports_collections() and \
+                     prefs['preserve_user_collections']):
                 flags |= Qt.ItemIsEditable
         return flags
 
-
     def search(self, text, reset=True):
         if not text or not text.strip():
             self.map = list(range(len(self.db)))
@@ -970,8 +979,8 @@ def sizecmp(x, y):
             x, y = int(self.db[x].size), int(self.db[y].size)
             return cmp(x, y)
         def tagscmp(x, y):
-            x = ','.join(self.db[x].device_collections)
-            y = ','.join(self.db[y].device_collections)
+            x = ','.join(sorted(getattr(self.db[x], 'device_collections', []))).lower()
+            y = ','.join(sorted(getattr(self.db[y], 'device_collections', []))).lower()
             return cmp(x, y)
         def libcmp(x, y):
             x, y = self.db[x].in_library, self.db[y].in_library
@@ -1072,6 +1081,36 @@ def paths_for_db_ids(self, db_ids):
                 res.append((r,b))
         return res
 
+    def get_collections_with_ids(self):
+        collections = set()
+        for book in self.db:
+            if book.device_collections is not None:
+                collections.update(set(book.device_collections))
+        self.collections = []
+        result = []
+        for i,collection in enumerate(collections):
+            result.append((i, collection))
+            self.collections.append(collection)
+        return result
+
+    def rename_collection(self, old_id, new_name):
+        old_name = self.collections[old_id]
+        for book in self.db:
+            if book.device_collections is None:
+                continue
+            if old_name in book.device_collections:
+                book.device_collections.remove(old_name)
+                if new_name not in book.device_collections:
+                    book.device_collections.append(new_name)
+
+    def delete_collection_using_id(self, old_id):
+        old_name = self.collections[old_id]
+        for book in self.db:
+            if book.device_collections is None:
+                continue
+            if old_name in book.device_collections:
+                book.device_collections.remove(old_name)
+
     def indices(self, rows):
         '''
         Return indices into underlying database from rows
@@ -1102,6 +1141,7 @@ def data(self, index, role):
             elif cname == 'collections':
                 tags = self.db[self.map[row]].device_collections
                 if tags:
+                    tags.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
                     return QVariant(', '.join(tags))
         elif role == Qt.ToolTipRole and index.isValid():
             if self.map[row] in self.indices_to_be_deleted():
@@ -1144,14 +1184,18 @@ def setData(self, index, value, role):
                 return False
             val = unicode(value.toString()).strip()
             idx = self.map[row]
+            if cname == 'collections':
+                tags = [i.strip() for i in val.split(',')]
+                tags = [t for t in tags if t]
+                self.db[idx].device_collections = tags
+                self.dataChanged.emit(index, index)
+                self.upload_collections.emit(self.db)
+                return True
+
             if cname == 'title' :
                 self.db[idx].title = val
             elif cname == 'authors':
                 self.db[idx].authors = string_to_authors(val)
-            elif cname == 'collections':
-                tags = [i.strip() for i in val.split(',')]
-                tags = [t for t in tags if t]
-                self.db[idx].device_collections = tags
             self.dataChanged.emit(index, index)
             self.booklist_dirtied.emit()
             done = True

src/calibre/gui2/library/views.py

@@ -15,7 +15,7 @@
     TextDelegate, DateDelegate, TagsDelegate, CcTextDelegate, \
     CcBoolDelegate, CcCommentsDelegate, CcDateDelegate
 from calibre.gui2.library.models import BooksModel, DeviceBooksModel
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.gui2 import error_dialog, gprefs
 from calibre.gui2.library import DEFAULT_SORT
 
@@ -347,7 +347,7 @@ def database_changed(self, db):
                     self.setItemDelegateForColumn(cm.index(colhead), delegate)
                 elif cc['datatype'] == 'comments':
                     self.setItemDelegateForColumn(cm.index(colhead), self.cc_comments_delegate)
-                elif cc['datatype'] == 'text':
+                elif cc['datatype'] in ('text', 'series'):
                     if cc['is_multiple']:
                         self.setItemDelegateForColumn(cm.index(colhead), self.tags_delegate)
                     else:
@@ -371,7 +371,8 @@ def database_changed(self, db):
     # Context Menu {{{
     def set_context_menu(self, edit_metadata, send_to_device, convert, view,
                          save, open_folder, book_details, delete,
-                         similar_menu=None, add_to_library=None):
+                         similar_menu=None, add_to_library=None,
+                         edit_device_collections=None):
         self.setContextMenuPolicy(Qt.DefaultContextMenu)
         self.context_menu = QMenu(self)
         if edit_metadata is not None:
@@ -393,6 +394,10 @@ def set_context_menu(self, edit_metadata, send_to_device, convert, view,
         if add_to_library is not None:
             func = partial(add_to_library[1], view=self)
             self.context_menu.addAction(add_to_library[0], func)
+        if edit_device_collections is not None:
+            func = partial(edit_device_collections[1], view=self)
+            self.edit_collections_menu = \
+                self.context_menu.addAction(edit_device_collections[0], func)
 
     def contextMenuEvent(self, event):
         self.context_menu.popup(event.globalPos())
@@ -494,6 +499,13 @@ def __init__(self, parent):
         self.setDragDropMode(self.NoDragDrop)
         self.setAcceptDrops(False)
 
+    def contextMenuEvent(self, event):
+        self.edit_collections_menu.setVisible(
+                            self._model.db.supports_collections() and \
+                            prefs['preserve_user_collections'])
+        self.context_menu.popup(event.globalPos())
+        event.accept()
+
     def set_database(self, db):
         self._model.set_database(db)
         self.restore_state()
@@ -505,6 +517,9 @@ def resizeColumnsToContents(self):
     def connect_dirtied_signal(self, slot):
         self._model.booklist_dirtied.connect(slot)
 
+    def connect_upload_collections_signal(self, func=None, oncard=None):
+        self._model.upload_collections.connect(partial(func, view=self, oncard=oncard))
+
     def dropEvent(self, *args):
         error_dialog(self, _('Not allowed'),
         _('Dropping onto a device is not supported. First add the book to the calibre library.')).exec_()

src/calibre/gui2/pictureflow/pictureflow.cpp

@@ -75,10 +75,6 @@
 
 #include <QDebug>
 
-// uncomment this to enable bilinear filtering for texture mapping
-// gives much better rendering, at the cost of memory space
-// #define PICTUREFLOW_BILINEAR_FILTER
-
 // for fixed-point arithmetic, we need minimum 32-bit long
 // long long (64-bit) might be useful for multiplication and division
 typedef long PFreal;
@@ -376,7 +372,6 @@ private:
   int slideWidth;
   int slideHeight;
   int fontSize;
-  int zoom;
   int queueLength;
 
   int centerIndex;
@@ -401,6 +396,7 @@ private:
 
   void recalc(int w, int h);
   QRect renderSlide(const SlideInfo &slide, int alpha=256, int col1=-1, int col=-1);
+  QRect renderCenterSlide(const SlideInfo &slide);
   QImage* surface(int slideIndex);
   void triggerRender();
   void resetSlides();
@@ -414,7 +410,6 @@ PictureFlowPrivate::PictureFlowPrivate(PictureFlow* w, int queueLength_)
   slideWidth = 200;
   slideHeight = 200;
   fontSize = 10;
-  zoom = 100;
 
   centerIndex = 0;
   queueLength = queueLength_;
@@ -464,21 +459,6 @@ void PictureFlowPrivate::setSlideSize(QSize size)
   triggerRender();
 }
 
-int PictureFlowPrivate::zoomFactor() const
-{
-  return zoom;
-}
-
-void PictureFlowPrivate::setZoomFactor(int z)
-{
-  if(z <= 0)
-    return;
-
-  zoom = z;
-  recalc(buffer.width(), buffer.height());
-  triggerRender();
-}
-
 QImage PictureFlowPrivate::slide(int index) const
 {
   return slideImages->image(index);
@@ -554,7 +534,8 @@ void PictureFlowPrivate::resize(int w, int h)
   if (w < 10) w = 10;
   if (h < 10) h = 10;
   slideHeight = int(float(h)/REFLECTION_FACTOR);
-  slideWidth = int(float(slideHeight) * 2/3.);
+  slideWidth = int(float(slideHeight) * 3./4.);
+  //qDebug() << slideHeight << "x" << slideWidth;
   fontSize = MAX(int(h/15.), 12);
   recalc(w, h);
   resetSlides();
@@ -595,15 +576,12 @@ void PictureFlowPrivate::resetSlides()
   }
 }
 
-#define BILINEAR_STRETCH_HOR 4
-#define BILINEAR_STRETCH_VER 4
-
 static QImage prepareSurface(QImage img, int w, int h)
 {
   Qt::TransformationMode mode = Qt::SmoothTransformation;
-  img = img.scaled(w, h, Qt::IgnoreAspectRatio, mode);
+  img = img.scaled(w, h, Qt::KeepAspectRatioByExpanding, mode);
 
-  // slightly larger, to accomodate for the reflection
+  // slightly larger, to accommodate for the reflection
   int hs = int(h * REFLECTION_FACTOR);
   int hofs = 0;
 
@@ -633,12 +611,6 @@ static QImage prepareSurface(QImage img, int w, int h)
       result.setPixel(h+hofs+y, x, qRgb(r, g, b));
     }
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-  int hh = BILINEAR_STRETCH_VER*hs;
-  int ww = BILINEAR_STRETCH_HOR*w;
-  result = result.scaled(hh, ww, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
-#endif
-
   return result;
 }
 
@@ -699,8 +671,12 @@ void PictureFlowPrivate::render()
 
   int nleft = leftSlides.count();
   int nright = rightSlides.count();
+  QRect r;
 
-  QRect r = renderSlide(centerSlide);
+  if (step == 0) 
+      r = renderCenterSlide(centerSlide);
+  else
+      r = renderSlide(centerSlide);
   int c1 = r.left();
   int c2 = r.right();
 
@@ -813,7 +789,23 @@ static inline uint BYTE_MUL_RGB16_32(uint x, uint a) {
     return t;
 }
 
+QRect PictureFlowPrivate::renderCenterSlide(const SlideInfo &slide) {
+  QImage* src = surface(slide.slideIndex);
+  if(!src)
+    return QRect();
 
+  int sw = src->height();
+  int sh = src->width();
+  int h = buffer.height();
+  QRect rect(buffer.width()/2 - sw/2, 0, sw, h-1);
+  int left = rect.left();
+
+  for(int x = 0; x < sh-1; x++)
+    for(int y = 0; y < sw; y++)
+      buffer.setPixel(left + y, 1+x, src->pixel(x, y));
+
+  return rect;
+}
 // Renders a slide to offscreen buffer. Returns a rect of the rendered area.
 // alpha=256 means normal, alpha=0 is fully black, alpha=128 half transparent
 // col1 and col2 limit the column for rendering.
@@ -826,13 +818,8 @@ int col1, int col2)
 
   QRect rect(0, 0, 0, 0);  
   
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-  int sw = src->height() / BILINEAR_STRETCH_HOR;
-  int sh = src->width() / BILINEAR_STRETCH_VER;
-#else
   int sw = src->height();
   int sh = src->width();
-#endif
   int h = buffer.height();
   int w = buffer.width();
 
@@ -848,7 +835,7 @@ int col1, int col2)
   col1 = qMin(col1, w-1);
   col2 = qMin(col2, w-1);
 
-  int distance = h * 100 / zoom;
+  int distance = h;
   PFreal sdx = fcos(slide.angle);
   PFreal sdy = fsin(slide.angle);
   PFreal xs = slide.cx - slideWidth * sdx/2;
@@ -878,15 +865,9 @@ int col1, int col2)
     PFreal hitx = fmul(dist, rays[x]);
     PFreal hitdist = fdiv(hitx - slide.cx, sdx);
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-    int column = sw*BILINEAR_STRETCH_HOR/2 + (hitdist*BILINEAR_STRETCH_HOR >> PFREAL_SHIFT);
-    if(column >= sw*BILINEAR_STRETCH_HOR)
-      break;
-#else
     int column = sw/2 + (hitdist >> PFREAL_SHIFT);
     if(column >= sw)
       break;
-#endif
     if(column < 0)
       continue;
 
@@ -901,13 +882,8 @@ int col1, int col2)
     QRgb565* pixel2 = (QRgb565*)(buffer.scanLine(y2)) + x;
     int pixelstep = pixel2 - pixel1;
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-    int center = (sh*BILINEAR_STRETCH_VER/2);
-    int dy = dist*BILINEAR_STRETCH_VER / h;
-#else
     int center = sh/2;
     int dy = dist / h;
-#endif
     int p1 = center*PFREAL_ONE - dy/2;
     int p2 = center*PFREAL_ONE + dy/2;
 
@@ -1155,16 +1131,6 @@ void PictureFlow::setSlideSize(QSize size)
   d->setSlideSize(size);
 }
 
-int PictureFlow::zoomFactor() const
-{
-  return d->zoomFactor();
-}
-
-void PictureFlow::setZoomFactor(int z)
-{
-  d->setZoomFactor(z);
-}
-
 QImage PictureFlow::slide(int index) const
 {
   return d->slide(index);

src/calibre/gui2/pictureflow/pictureflow.h

@@ -91,7 +91,6 @@ Q_OBJECT
 
   Q_PROPERTY(int currentSlide READ currentSlide WRITE setCurrentSlide)
   Q_PROPERTY(QSize slideSize READ slideSize WRITE setSlideSize)
-  Q_PROPERTY(int zoomFactor READ zoomFactor WRITE setZoomFactor)
 
 public:
   /*!
@@ -120,16 +119,6 @@ public:
   */  
   void setSlideSize(QSize size);
 
-  /*!
-    Sets the zoom factor (in percent).
-  */ 
-  void setZoomFactor(int zoom);
-
-  /*!
-    Returns the zoom factor (in percent).
-  */
-  int zoomFactor() const;
-
   /*!
     Clears any caches held to free up memory
   */

src/calibre/gui2/pictureflow/pictureflow.sip

@@ -40,10 +40,6 @@ public :
 
   void setSlideSize(QSize size);
 
-  void setZoomFactor(int zoom);
-
-  int zoomFactor() const;
-
   void clearCaches();
 
   virtual QImage slide(int index) const;

src/calibre/gui2/search_box.py

@@ -56,7 +56,8 @@ class SearchBox2(QComboBox):
     To use this class:
 
         * Call initialize()
-        * Connect to the search() and cleared() signals from this widget
+        * Connect to the search() and cleared() signals from this widget.
+        * Connect to the cleared() signal to know when the box content changes
         * Call search_done() after every search is complete
         * Use clear() to clear back to the help message
     '''
@@ -75,6 +76,7 @@ def __init__(self, parent=None):
                 type=Qt.DirectConnection)
         self.line_edit.mouse_released.connect(self.mouse_released,
                 type=Qt.DirectConnection)
+        self.activated.connect(self.history_selected)
         self.setEditable(True)
         self.help_state = False
         self.as_you_type = True
@@ -139,6 +141,9 @@ def search_done(self, ok):
 
     def key_pressed(self, event):
         self.normalize_state()
+        if self._in_a_search:
+            self.emit(SIGNAL('changed()'))
+            self._in_a_search = False
         if event.key() in (Qt.Key_Return, Qt.Key_Enter):
             self.do_search()
         self.timer = self.startTimer(self.__class__.INTERVAL)
@@ -154,6 +159,10 @@ def timerEvent(self, event):
             self.timer = None
             self.do_search()
 
+    def history_selected(self, text):
+        self.emit(SIGNAL('changed()'))
+        self.do_search()
+
     @property
     def smart_text(self):
         text = unicode(self.currentText()).strip()
@@ -345,6 +354,7 @@ def __init__(self):
         self.search.initialize('main_search_history', colorize=True,
                 help_text=_('Search (For Advanced Search click the button to the left)'))
         self.connect(self.search, SIGNAL('cleared()'), self.search_box_cleared)
+        self.connect(self.search, SIGNAL('changed()'), self.search_box_changed)
         self.connect(self.clear_button, SIGNAL('clicked()'), self.search.clear)
         QObject.connect(self.advanced_search_button, SIGNAL('clicked(bool)'),
                         self.do_advanced_search)
@@ -364,6 +374,9 @@ def search_box_cleared(self):
         self.saved_search.clear_to_help()
         self.set_number_of_books_shown()
 
+    def search_box_changed(self):
+        self.tags_view.clear()
+
     def do_advanced_search(self, *args):
         d = SearchDialog(self)
         if d.exec_() == QDialog.Accepted:

src/calibre/gui2/tag_view.py

@@ -15,6 +15,7 @@
                      QAbstractItemModel, QVariant, QModelIndex, QMenu, \
                      QPushButton, QWidget, QItemDelegate
 
+from calibre.ebooks.metadata import title_sort
 from calibre.gui2 import config, NONE
 from calibre.utils.config import prefs
 from calibre.library.field_metadata import TagsIcons
@@ -680,9 +681,50 @@ def do_user_categories_edit(self, on_category=None):
             self.tags_view.recount()
 
     def do_tags_list_edit(self, tag, category):
-        d = TagListEditor(self, self.library_view.model().db, tag, category)
+        db=self.library_view.model().db
+        if category == 'tags':
+            result = db.get_tags_with_ids()
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        elif category == 'series':
+            result = db.get_series_with_ids()
+            compare = (lambda x,y:cmp(title_sort(x).lower(), title_sort(y).lower()))
+        elif category == 'publisher':
+            result = db.get_publishers_with_ids()
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        else: # should be a custom field
+            cc_label = None
+            if category in db.field_metadata:
+                cc_label = db.field_metadata[category]['label']
+                result = self.db.get_custom_items_with_ids(label=cc_label)
+            else:
+                result = []
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+
+        d = TagListEditor(self, tag_to_match=tag, data=result, compare=compare)
         d.exec_()
         if d.result() == d.Accepted:
+            to_rename = d.to_rename # dict of new text to old id
+            to_delete = d.to_delete # list of ids
+            rename_func = None
+            if category == 'tags':
+                rename_func = db.rename_tag
+                delete_func = db.delete_tag_using_id
+            elif category == 'series':
+                rename_func = db.rename_series
+                delete_func = db.delete_series_using_id
+            elif category == 'publisher':
+                rename_func = db.rename_publisher
+                delete_func = db.delete_publisher_using_id
+            else:
+                rename_func = partial(db.rename_custom_item, label=cc_label)
+                delete_func = partial(db.delete_custom_item_using_id, label=cc_label)
+            if rename_func:
+                for text in to_rename:
+                        for old_id in to_rename[text]:
+                            rename_func(old_id, new_name=unicode(text))
+                for item in to_delete:
+                    delete_func(item)
+
             # Clean up everything, as information could have changed for many books.
             self.library_view.model().refresh()
             self.tags_view.set_new_model()

src/calibre/gui2/ui.py

@@ -473,6 +473,8 @@ def location_selected(self, location):
             self.search_restriction.setEnabled(False)
             for action in list(self.delete_menu.actions())[1:]:
                 action.setEnabled(False)
+            # Reset the view in case something changed while it was invisible
+            self.current_view().reset()
         self.set_number_of_books_shown()
 
 

src/calibre/gui2/widgets.py

@@ -957,16 +957,19 @@ def __init__(self, icon, text, splitter, parent=None):
 
         self.splitter = splitter
         splitter.state_changed.connect(self.update_state)
+        self.setCursor(Qt.PointingHandCursor)
 
     def set_state_to_show(self, *args):
         self.setChecked(False)
         label =_('Show')
         self.setText(label + ' ' + self.label)
+        self.setToolTip(self.text())
 
     def set_state_to_hide(self, *args):
         self.setChecked(True)
         label = _('Hide')
         self.setText(label + ' ' + self.label)
+        self.setToolTip(self.text())
 
     def update_state(self, *args):
         if self.splitter.is_side_index_hidden:

src/calibre/library/caches.py

@@ -401,7 +401,8 @@ def get_matches(self, location, query):
             for x in self.field_metadata:
                 if len(self.field_metadata[x]['search_terms']):
                     db_col[x] = self.field_metadata[x]['rec_index']
-                    if self.field_metadata[x]['datatype'] not in ['text', 'comments']:
+                    if self.field_metadata[x]['datatype'] not in \
+                                                ['text', 'comments', 'series']:
                         exclude_fields.append(db_col[x])
                     col_datatype[db_col[x]] = self.field_metadata[x]['datatype']
                     is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple']
@@ -580,16 +581,18 @@ def refresh(self, db, field=None, ascending=True):
             self.sort(field, ascending)
         self._map_filtered = list(self._map)
 
-    def seriescmp(self, x, y):
-        sidx = self.FIELD_MAP['series']
+    def seriescmp(self, sidx, siidx, x, y, library_order=None):
         try:
-            ans = cmp(title_sort(self._data[x][sidx].lower()),
-                    title_sort(self._data[y][sidx].lower()))
+            if library_order:
+                ans = cmp(title_sort(self._data[x][sidx].lower()),
+                                title_sort(self._data[y][sidx].lower()))
+            else:
+                ans = cmp(self._data[x][sidx].lower(),
+                                                self._data[y][sidx].lower())
         except AttributeError: # Some entries may be None
             ans = cmp(self._data[x][sidx], self._data[y][sidx])
         if ans != 0: return ans
-        sidx = self.FIELD_MAP['series_index']
-        return cmp(self._data[x][sidx], self._data[y][sidx])
+        return cmp(self._data[x][siidx], self._data[y][siidx])
 
     def cmp(self, loc, x, y, asstr=True, subsort=False):
         try:
@@ -617,18 +620,27 @@ def sort(self, field, ascending, subsort=False):
         elif field == 'title': field = 'sort'
         elif field == 'authors': field = 'author_sort'
         as_string = field not in ('size', 'rating', 'timestamp')
-        if self.field_metadata[field]['is_custom']:
-            as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
-            field = self.field_metadata[field]['colnum']
 
         if self.first_sort:
             subsort = True
             self.first_sort = False
-        fcmp = self.seriescmp \
-                if field == 'series' and \
-                   tweaks['title_series_sorting'] == 'library_order' \
-                else \
-                   functools.partial(self.cmp, self.FIELD_MAP[field],
+        if self.field_metadata[field]['is_custom']:
+            if self.field_metadata[field]['datatype'] == 'series':
+                fcmp = functools.partial(self.seriescmp,
+                    self.field_metadata[field]['rec_index'],
+                    self.field_metadata.cc_series_index_column_for(field),
+                    library_order=tweaks['title_series_sorting'] == 'library_order')
+            else:
+                as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
+                field = self.field_metadata[field]['colnum']
+                fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
+                                     subsort=subsort, asstr=as_string)
+        elif field == 'series':
+            fcmp = functools.partial(self.seriescmp, self.FIELD_MAP['series'],
+                self.FIELD_MAP['series_index'],
+                library_order=tweaks['title_series_sorting'] == 'library_order')
+        else:
+            fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
                                      subsort=subsort, asstr=as_string)
         self._map.sort(cmp=fcmp, reverse=not ascending)
         self._map_filtered = [id for id in self._map if id in self._map_filtered]

src/calibre/library/cli.py

@@ -7,11 +7,11 @@
 Command line interface to the calibre database.
 '''
 
-import sys, os, cStringIO
+import sys, os, cStringIO, re
 from textwrap import TextWrapper
 
 from calibre import terminal_controller, preferred_encoding, prints
-from calibre.utils.config import OptionParser, prefs
+from calibre.utils.config import OptionParser, prefs, tweaks
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.library.database2 import LibraryDatabase2
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
@@ -680,9 +680,31 @@ def command_catalog(args, dbpath):
 
 # end of GR additions
 
+def parse_series_string(db, label, value):
+    val = unicode(value).strip()
+    s_index = None
+    pat = re.compile(r'\[([.0-9]+)\]')
+    match = pat.search(val)
+    if match is not None:
+        val = pat.sub('', val).strip()
+        s_index = float(match.group(1))
+    elif val:
+        if tweaks['series_index_auto_increment'] == 'next':
+            s_index = db.get_next_cc_series_num_for(val, label=label)
+        else:
+            s_index = 1.0
+    return val, s_index
+
 def do_set_custom(db, col, id_, val, append):
-    db.set_custom(id_, val, label=col, append=append)
-    prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True))
+    if db.custom_column_label_map[col]['datatype'] == 'series':
+        val, s_index = parse_series_string(db, col, val)
+        db.set_custom(id_, val, extra=s_index, label=col, append=append)
+        prints('Data set to: %r[%4.2f]'%
+               (db.get_custom(id_, label=col, index_is_id=True),
+                db.get_custom_extra(id_, label=col, index_is_id=True)))
+    else:
+        db.set_custom(id_, val, label=col, append=append)
+        prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True))
 
 def set_custom_option_parser():
     parser = get_parser(_(

src/calibre/library/custom_columns.py

@@ -8,6 +8,7 @@
 
 import json
 from functools import partial
+from math import floor
 
 from calibre import prints
 from calibre.constants import preferred_encoding
@@ -16,7 +17,7 @@
 class CustomColumns(object):
 
     CUSTOM_DATA_TYPES = frozenset(['rating', 'text', 'comments', 'datetime',
-        'int', 'float', 'bool'])
+        'int', 'float', 'bool', 'series'])
 
     def custom_table_names(self, num):
         return 'custom_column_%d'%num, 'books_custom_column_%d_link'%num
@@ -137,7 +138,8 @@ def adapt_bool(x, d):
                 'bool':  adapt_bool,
                 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),
                 'datetime' : adapt_datetime,
-                'text':adapt_text
+                'text':adapt_text,
+                'series':adapt_text
         }
 
         # Create Tag Browser categories for custom columns
@@ -171,6 +173,19 @@ def get_custom(self, idx, label=None, num=None, index_is_id=False):
                 ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
         return ans
 
+    def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
+        if label is not None:
+            data = self.custom_column_label_map[label]
+        if num is not None:
+            data = self.custom_column_num_map[num]
+        # add future datatypes with an extra column here
+        if data['datatype'] not in ['series']:
+            return None
+        ign,lt = self.custom_table_names(data['num'])
+        idx = idx if index_is_id else self.id(idx)
+        return self.conn.get('''SELECT extra FROM %s
+                                WHERE book=?'''%lt, (idx,), all=False)
+
     # convenience methods for tag editing
     def get_custom_items_with_ids(self, label=None, num=None):
         if label is not None:
@@ -220,6 +235,28 @@ def delete_custom_item_using_id(self, id, label=None, num=None):
             self.conn.commit()
     # end convenience methods
 
+    def get_next_cc_series_num_for(self, series, label=None, num=None):
+        if label is not None:
+            data = self.custom_column_label_map[label]
+        if num is not None:
+            data = self.custom_column_num_map[num]
+        if data['datatype'] != 'series':
+            return None
+        table, lt = self.custom_table_names(data['num'])
+        # get the id of the row containing the series string
+        series_id = self.conn.get('SELECT id from %s WHERE value=?'%table,
+                                                        (series,), all=False)
+        if series_id is None:
+            return 1.0
+        # get the label of the associated series number table
+        series_num = self.conn.get('''
+                SELECT MAX({lt}.extra) FROM {lt}
+                WHERE {lt}.book IN (SELECT book FROM {lt} where value=?)
+                '''.format(lt=lt), (series_id,), all=False)
+        if series_num is None:
+            return 1.0
+        return floor(series_num+1)
+
     def all_custom(self, label=None, num=None):
         if label is not None:
             data = self.custom_column_label_map[label]
@@ -271,9 +308,8 @@ def set_custom_column_metadata(self, num, name=None, label=None,
             self.conn.commit()
         return changed
 
-
-
-    def set_custom(self, id_, val, label=None, num=None, append=False, notify=True):
+    def set_custom(self, id_, val, label=None, num=None,
+                   append=False, notify=True, extra=None):
         if label is not None:
             data = self.custom_column_label_map[label]
         if num is not None:
@@ -317,10 +353,17 @@ def set_custom(self, id_, val, label=None, num=None, append=False, notify=True):
                             'INSERT INTO %s(value) VALUES(?)'%table, (x,)).lastrowid
                 if not self.conn.get(
                     'SELECT book FROM %s WHERE book=? AND value=?'%lt,
-                    (id_, xid), all=False):
-                    self.conn.execute(
-                        'INSERT INTO %s(book, value) VALUES (?,?)'%lt,
-                            (id_, xid))
+                                                        (id_, xid), all=False):
+                    if data['datatype'] == 'series':
+                        self.conn.execute(
+                            '''INSERT INTO %s(book, value, extra)
+                               VALUES (?,?,?)'''%lt, (id_, xid, extra))
+                        self.data.set(id_, self.FIELD_MAP[data['num']]+1,
+                                      extra, row_is_id=True)
+                    else:
+                        self.conn.execute(
+                            '''INSERT INTO %s(book, value)
+                                VALUES (?,?)'''%lt, (id_, xid))
             self.conn.commit()
             nval = self.conn.get(
                     'SELECT custom_%s FROM meta2 WHERE id=?'%data['num'],
@@ -370,6 +413,9 @@ def custom_columns_in_meta(self):
                     {table} ON(link.value={table}.id) WHERE link.book=books.id)
                     custom_{num}
                 '''.format(query=query%table, lt=lt, table=table, num=data['num'])
+                if data['datatype'] == 'series':
+                    line += ''',(SELECT extra FROM {lt} WHERE {lt}.book=books.id)
+                        custom_index_{num}'''.format(lt=lt, num=data['num'])
             else:
                 line = '''
                 (SELECT value FROM {table} WHERE book=books.id) custom_{num}
@@ -393,7 +439,7 @@ def create_custom_column(self, label, name, datatype, is_multiple,
 
         if datatype in ('rating', 'int'):
             dt = 'INT'
-        elif datatype in ('text', 'comments'):
+        elif datatype in ('text', 'comments', 'series'):
             dt = 'TEXT'
         elif datatype in ('float',):
             dt = 'REAL'
@@ -404,6 +450,10 @@ def create_custom_column(self, label, name, datatype, is_multiple,
         collate = 'COLLATE NOCASE' if dt == 'TEXT' else ''
         table, lt = self.custom_table_names(num)
         if normalized:
+            if datatype == 'series':
+                s_index = 'extra REAL,'
+            else:
+                s_index = ''
             lines = [
                 '''\
                 CREATE TABLE %s(
@@ -419,8 +469,9 @@ def create_custom_column(self, label, name, datatype, is_multiple,
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     book INTEGER NOT NULL,
                     value INTEGER NOT NULL,
+                    %s
                     UNIQUE(book, value)
-                    );'''%lt,
+                    );'''%(lt, s_index),
 
                 'CREATE INDEX %s_aidx ON %s (value);'%(lt,lt),
                 'CREATE INDEX %s_bidx ON %s (book);'%(lt,lt),
@@ -468,7 +519,7 @@ def create_custom_column(self, label, name, datatype, is_multiple,
                           ratings as r
                      WHERE {lt}.value={table}.id and bl.book={lt}.book and
                            r.id = bl.rating and r.rating <> 0) avg_rating,
-                     value AS sort
+                    value AS sort
                 FROM {table};
 
                 CREATE VIEW tag_browser_filtered_{table} AS SELECT
@@ -483,7 +534,7 @@ def create_custom_column(self, label, name, datatype, is_multiple,
                      WHERE {lt}.value={table}.id AND bl.book={lt}.book AND
                            r.id = bl.rating AND r.rating <> 0 AND
                            books_list_filter(bl.book)) avg_rating,
-                     value AS sort
+                    value AS sort
                 FROM {table};
 
                 '''.format(lt=lt, table=table),

src/calibre/library/database2.py

@@ -237,6 +237,11 @@ def initialize_dynamic(self):
                                         self.custom_column_num_map[col]['label'],
                                         base,
                                         prefer_custom=True)
+            if self.custom_column_num_map[col]['datatype'] == 'series':
+                # account for the series index column. Field_metadata knows that
+                # the series index is one larger than the series. If you change
+                # it here, be sure to change it there as well.
+                self.FIELD_MAP[str(col)+'_s_index'] = base = base+1
 
         self.FIELD_MAP['cover'] = base+1
         self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False)
@@ -777,6 +782,15 @@ def get_categories(self, sort='name', ids=None, icon_map=None):
                                         icon=icon, tooltip=tooltip)
                                     for r in data if item_not_zero_func(r)]
 
+        # Needed for legacy databases that have multiple ratings that
+        # map to n stars
+        for r in categories['rating']:
+            for x in categories['rating']:
+                if r.name == x.name and r.id != x.id:
+                    r.count = r.count + x.count
+                    categories['rating'].remove(x)
+                    break
+
         # We delayed computing the standard formats category because it does not
         # use a view, but is computed dynamically
         categories['formats'] = []

src/calibre/library/field_metadata.py

@@ -81,7 +81,7 @@ class FieldMetadata(dict):
                            'column':'name',
                            'link_column':'series',
                            'category_sort':'(title_sort(name))',
-                           'datatype':'text',
+                           'datatype':'series',
                            'is_multiple':None,
                            'kind':'field',
                            'name':_('Series'),
@@ -398,6 +398,8 @@ def remove_dynamic_categories(self):
             if val['is_category'] and val['kind'] in ('user', 'search'):
                 del self._tb_cats[key]
 
+    def cc_series_index_column_for(self, key):
+        return self._tb_cats[key]['rec_index'] + 1
 
     def add_user_category(self, label, name):
         if label in self._tb_cats:

src/calibre/manual/conf.py

@@ -100,9 +100,8 @@
 html_title       = 'calibre User Manual'
 html_short_title = 'Start'
 html_logo        = 'resources/logo.png'
-epub_titlepage   = 'resources/titlepage.html'
-epub_logo        = 'resources/logo.png'
 epub_author      = 'Kovid Goyal'
+epub_cover       = 'resources/epub_cover.jpg'
 
 # Custom sidebar templates, maps document names to template names.
 #html_sidebars = {}

src/calibre/manual/custom.py

@@ -304,9 +304,8 @@ def auto_member(dirname, arguments, options, content, lineno,
     return list(node)
 
 def setup(app):
-    app.add_config_value('epub_titlepage', None, False)
+    app.add_config_value('epub_cover', None, False)
     app.add_config_value('epub_author', '', False)
-    app.add_config_value('epub_logo', None, False)
     app.add_builder(CustomBuilder)
     app.add_builder(CustomQtBuild)
     app.add_builder(EPUBHelpBuilder)

src/calibre/manual/epub.py

@@ -50,6 +50,7 @@
         <dc:identifier opf:scheme="sphinx" id="sphinx_id">{uid}</dc:identifier>
         <dc:date>{date}</dc:date>
         <meta name="calibre:publication_type" content="sphinx_manual" />
+        <meta name="cover" content="cover"/>
     </metadata>
     <manifest>
     {manifest}
@@ -71,6 +72,29 @@
    </rootfiles>
 </container>
 '''
+
+SVG_TEMPLATE = '''\
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+        <meta name="calibre:cover" content="true" />
+        <title>Cover</title>
+        <style type="text/css" title="override_css">
+            @page {padding: 0pt; margin:0pt}
+            body { text-align: center; padding:0pt; margin: 0pt; }
+        </style>
+    </head>
+    <body>
+        <svg version="1.1" xmlns="http://www.w3.org/2000/svg"
+            xmlns:xlink="http://www.w3.org/1999/xlink"
+            width="100%%" height="100%%" viewBox="0 0 600 800"
+            preserveAspectRatio="none">
+            <image width="600" height="800" xlink:href="%s"/>
+        </svg>
+    </body>
+</html>
+'''
+
 class TOC(list):
 
     def __init__(self, title=None, href=None):
@@ -151,8 +175,6 @@ def render_opf(self):
         spine = [' '*8+'<itemref idref=%s />'%quoteattr(x) for x in self.spine]
         spine = '\n'.join(spine)
         guide = ''
-        if self.conf.epub_titlepage:
-            guide = ' '*8 + '<reference type="cover"  href="_static/titlepage.html" />'
 
         opf = OPF.format(title=escape(self.conf.html_title),
                 author=escape(self.conf.epub_author), uid=str(uuid.uuid4()),
@@ -162,18 +184,15 @@ def render_opf(self):
         self.manifest['content.opf'] = ('application/oebps-package+xml', 'opf')
 
     def create_titlepage(self):
-        if self.conf.epub_titlepage:
-            img = ''
-            if self.conf.epub_logo:
-                img = '_static/epub_logo'+os.path.splitext(self.conf.epub_logo)[1]
-                shutil.copyfile(self.conf.epub_logo,
-                        os.path.join(self.html_outdir, *img.split('/')))
-            raw = open(self.conf.epub_titlepage, 'rb').read()
-            raw = raw%dict(title=self.conf.html_title,
-                    version=self.conf.version,
-                    img=img.split('/')[-1],
-                    author=self.conf.epub_author)
-            open(os.path.join(self.html_outdir, '_static', 'titlepage.html'), 'wb').write(raw)
+        self.cover_image_url = None
+        if self.conf.epub_cover:
+            img = '_static/'+os.path.basename(self.conf.epub_cover)
+            shutil.copyfile(self.conf.epub_cover, os.path.join(self.html_outdir,
+                *img.split('/')))
+            self.cover_image_url = img
+            tp = SVG_TEMPLATE%img.split('/')[-1]
+            open(os.path.join(self.html_outdir, '_static', 'titlepage.html'),
+                    'wb').write(tp)
 
     def generate_manifest(self):
         self.manifest = {}
@@ -190,8 +209,12 @@ def generate_manifest(self):
                     self.manifest[url] = 'application/octet-stream'
                 if self.manifest[url] == 'text/html':
                     self.manifest[url] = 'application/xhtml+xml'
-                self.manifest[url] = (self.manifest[url], 'id'+str(id))
-                id += 1
+                if self.cover_image_url and url.endswith(self.cover_image_url):
+                    id_ = 'cover'
+                else:
+                    id_ = 'id'+str(id)
+                    id += 1
+                self.manifest[url] = (self.manifest[url], id_)
 
     def isdocnode(self, node):
         if not isinstance(node, nodes.list_item):
@@ -227,7 +250,7 @@ def generate_toc(self):
         open('toc.ncx', 'wb').write(ncx)
         self.manifest['toc.ncx'] = ('application/x-dtbncx+xml', 'ncx')
         self.spine.insert(0, self.manifest[self.conf.master_doc+'.html'][1])
-        if self.conf.epub_titlepage:
+        if self.conf.epub_cover:
             self.spine.insert(0, self.manifest['_static/titlepage.html'][1])
 
     def add_to_spine(self, href):

src/calibre/manual/faq.rst

@@ -7,7 +7,7 @@ Frequently Asked Questions
 
 .. contents:: Contents
   :depth: 1
-  :local: 
+  :local:
 
 E-book Format Conversion
 -------------------------
@@ -30,7 +30,7 @@ It can convert every input format in the following list, to every output format.
 
 What are the best source formats to convert?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-In order of decreasing preference: LIT, MOBI, EPUB, HTML, PRC, RTF, PDB, TXT, PDF 
+In order of decreasing preference: LIT, MOBI, EPUB, HTML, PRC, RTF, PDB, TXT, PDF
 
 Why does the PDF conversion lose some images/tables?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -40,7 +40,7 @@ are also represented as vector diagrams, thus they cannot be extracted.
 How do I convert a collection of HTML files in a specific order?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 In order to convert a collection of HTML files in a specific oder, you have to create a table of contents file. That is, another HTML file that contains links to all the other files in the desired order. Such a file looks like::
- 
+
    <html>
       <body>
         <h1>Table of Contents</h1>
@@ -60,16 +60,16 @@ Then just add this HTML file to the GUI and use the convert button to create you
 
 How do I convert my file containing non-English characters, or smart quotes?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-There are two aspects to this problem: 
+There are two aspects to this problem:
   1. Knowing the encoding of the source file: |app| tries to guess what character encoding your source files use, but often, this is impossible, so you need to tell it what encoding to use. This can be done in the GUI via the :guilabel:`Input character encoding` field in the :guilabel:`Look & Feel` section. The command-line tools all have an :option:`--input-encoding` option.
-  2. When adding HTML files to |app|, you may need to tell |app| what encoding the files are in. To do this go to Preferences->Plugins->File Type plugins and customize the HTML2Zip plugin, telling it what encoding your HTML files are in. Now when you add HTML files to |app| they will be correctly processed. HTML files from different sources often have different encodings, so you may have to change this setting repeatedly. A common encoding for many files from the web is ``cp1252`` and I would suggest you try that first. Note that when converting HTML files, leave the input encoding setting mentioned above blank. This is because the HTML2ZIP plugin automatically converts the HTML files to a standard encoding (utf-8). 
-  3. Embedding fonts: If you are generating an LRF file to read on your SONY Reader, you are limited by the fact that the Reader only supports a few non-English characters in the fonts it comes pre-loaded with. You can work around this problem by embedding a unicode-aware font that supports the character set your file uses into the LRF file. You should embed atleast a serif and a sans-serif font. Be aware that embedding fonts significantly slows down page-turn speed on the reader. 
+  2. When adding HTML files to |app|, you may need to tell |app| what encoding the files are in. To do this go to Preferences->Plugins->File Type plugins and customize the HTML2Zip plugin, telling it what encoding your HTML files are in. Now when you add HTML files to |app| they will be correctly processed. HTML files from different sources often have different encodings, so you may have to change this setting repeatedly. A common encoding for many files from the web is ``cp1252`` and I would suggest you try that first. Note that when converting HTML files, leave the input encoding setting mentioned above blank. This is because the HTML2ZIP plugin automatically converts the HTML files to a standard encoding (utf-8).
+  3. Embedding fonts: If you are generating an LRF file to read on your SONY Reader, you are limited by the fact that the Reader only supports a few non-English characters in the fonts it comes pre-loaded with. You can work around this problem by embedding a unicode-aware font that supports the character set your file uses into the LRF file. You should embed atleast a serif and a sans-serif font. Be aware that embedding fonts significantly slows down page-turn speed on the reader.
 
 
 How do I use some of the advanced features of the conversion tools?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- You can get help on any individual feature of the converters by mousing over it in the GUI or running ``ebook-convert dummy.html .epub -h`` at a terminal. A good place to start is to look at the following demo files that demonstrate some of the advanced features: 
-  * `html-demo.zip <http://calibre-ebook.com/downloads/html-demo.zip>`_ 
+ You can get help on any individual feature of the converters by mousing over it in the GUI or running ``ebook-convert dummy.html .epub -h`` at a terminal. A good place to start is to look at the following demo files that demonstrate some of the advanced features:
+  * `html-demo.zip <http://calibre-ebook.com/downloads/html-demo.zip>`_
 
 
 Device Integration
@@ -95,12 +95,39 @@ We just need some information from you:
     device supports SD cards, insert them. Then connect your device. In calibre go to Preferences->Advanced
     and click the "Debug device detection" button. This will create some debug output. Copy it to a file
     and repeat the process, this time with your device disconnected.
-  * Send both the above outputs to us with the other information and we will write a device driver for your 
+  * Send both the above outputs to us with the other information and we will write a device driver for your
     device.
 
 Once you send us the output for a particular operating system, support for the device in that operating system
 will appear in the next release of |app|.
 
+How does |app| manage collections on my SONY reader?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When |app| connects with the device, it retrieves all collections for the books on the device. The collections
+of which books are members are shown on the device view.
+
+When you send a book to the device, |app| will add the book to collections based on the metadata for that book. By
+default, collections are created from tags and series. You can control what metadata is used by going to
+Preferences->Plugins->Device Interface plugins and customizing the SONY device interface plugin. If you remove all
+values, |app| will not add the book to any collection.
+
+Collection management is largely controlled by 'Preserve device collections' found at Preferences->Add/Save->Sending
+to device. If checked (the default), managing collections is left to the user; |app| will not delete already
+existing collections for a book on your device when you resend the book to the device, but |app| will add the book to
+collections if necessary.  To ensure that the collections for a book are based only on current |app| metadata, first
+delete the books from the device, then resend the books.  You can edit collections directly on the device view by
+double-clicking or right-clicking in the collections column.
+
+If 'Preserve device collections' is not checked, then |app| will manage collections. Collections will be built using
+|app| metadata exclusively.  Sending a book to the device will correct the collections for that book so its
+collections exactly match the book's metadata. Collections are added and deleted as necessary.  Editing collections on
+the device pane is not permitted, because collections not in the metadata will be removed automatically.
+
+In summary, check 'Preserve device collections' if you want to manage collections yourself.  Collections for a book
+will never be removed by |app|, but can be removed by you by editing on the device view.  Uncheck 'Preserve device
+collections' if you want |app| to manage the collections, adding books to and removing books from collections as
+needed.  
 
 Can I use both |app| and the SONY software to manage my reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -130,7 +157,7 @@ simplest is to simply re-name the executable file that launches the library prog
 Can I use the collections feature of the SONY reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 |app| has full support for collections. When you add tags to a book's metadata, those tags are turned into collections when you upload the book to the SONY reader. Also, the series information is automatically
-turned into a collection on the reader. Note that the PRS-500 does not support collections for books stored on the SD card. The PRS-505 does. 
+turned into a collection on the reader. Note that the PRS-500 does not support collections for books stored on the SD card. The PRS-505 does.
 
 How do I use |app| with my iPad/iPhone/iTouch?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -139,7 +166,7 @@ The easiest way to browse your |app| collection on your Apple device (iPad/iPhon
 
 First perform the following steps in |app|
 
-  * Set the Preferred Output Format in |app| to EPUB (The output format can be set under Preferences->General) 
+  * Set the Preferred Output Format in |app| to EPUB (The output format can be set under Preferences->General)
   * Set the output profile to iPad (this will work for iPhone/iPods as well), under Preferences->Conversion->Page Setup
   * Convert the books you want to read on your iPhone to EPUB format by selecting them and clicking the Convert button.
   * Turn on the Content Server in |app|'s preferences and leave |app| running.
@@ -160,7 +187,7 @@ Alternative for the iPad
 As of |app| version 0.7.0, you can plugin your iPad into the computer using its charging cable, and |app| will detect it and show you a list of books on the iPad. You can then use the Send to device button to send books directly to iBooks on the iPad.
 
 This method only works on Windows XP and higher and OS X 10.5 and higher. Linux is not supported (iTunes is not available in linux) and OS X 10.4 is not supported. For more details, see
-`this forum post http://www.mobileread.com/forums/showpost.php?p=944079&postcount=1`_.
+`this forum post <http://www.mobileread.com/forums/showpost.php?p=944079&postcount=1>`_.
 
 How do I use |app| with my Android phone?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -171,7 +198,7 @@ Can I access my |app| books using the web browser in my Kindle or other reading
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 |app| has a *Content Server* that exports the books in |app| as a web page. You can turn it on under
-Preferences->Content Server. Then just point the web browser on your device to the computer running 
+Preferences->Content Server. Then just point the web browser on your device to the computer running
 the Content Server and you will be able to browse your book collection. For example, if the computer running
 the server has IP address 63.45.128.5, in the browser, you would type::
 
@@ -190,11 +217,16 @@ The most likely cause of this is your antivirus program. Try temporarily disabli
 Why is my device not detected in linux?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-|app| uses something called SYSFS to detect devices in linux. The linux kernel can export two version of SYSFS, one of which is deprecated. Some linux distributions still ship with kernels that support the deprecated version of SYSFS, even though it was deprecated a long time ago. In this case, device detection in |app| will not work. You can check what version of SYSFS is exported by your kernel with the following command::
-    
+|app| needs your linux kernel to have been setup correctly to detect devices. If your devices are not detected, perform the following tests::
+
     grep SYSFS_DEPRECATED /boot/config-`uname -r`
 
-You should see something like ``CONFIG_SYSFS_DEPRECATED_V2 is not set``. If you don't you have to either recompile your kernel with the correct setting, or upgrade your linux distro to a more modern version, where this will not be set.
+You should see something like ``CONFIG_SYSFS_DEPRECATED_V2 is not set``.
+Also, ::
+
+    grep CONFIG_SCSI_MULTI_LUN /boot/config-`uname -r`
+
+must return ``CONFIG_SCSI_MULTI_LUN=y``. If you don't see either, you have to recompile your kernel with the correct settings.
 
 Library Management
 ------------------
@@ -222,7 +254,7 @@ Now this makes it very easy to find for example all science fiction books by Isa
 
     ReadStatus -> Genre -> Author -> Series
 
-In |app|, you would instead use tags to mark genre and read status and then just use a simple search query like ``tag:scifi and not tag:read``. |app| even has a nice graphical interface, so you don't need to learn its search language instead you can just click on tags to include or exclude them from the search. 
+In |app|, you would instead use tags to mark genre and read status and then just use a simple search query like ``tag:scifi and not tag:read``. |app| even has a nice graphical interface, so you don't need to learn its search language instead you can just click on tags to include or exclude them from the search.
 
 Why doesn't |app| have a column for foo?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -230,7 +262,7 @@ Why doesn't |app| have a column for foo?
 
 How do I move my |app| library from one computer to another?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking Preferences. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder. 
+Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking Preferences. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder.
 
 Note that if you are transferring between different types of computers (for example Windows to OS X) then after doing the above you should also go to Preferences->Advanced and click the Check database integrity button. It will warn you about missing files, if any, which you should then transfer by hand.
 
@@ -241,11 +273,11 @@ Content From The Web
   :depth: 1
   :local:
 
-My downloaded news content causes the reader to reset. 
+My downloaded news content causes the reader to reset.
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 This is a bug in the SONY firmware. The problem can be mitigated by switching the output format to EPUB
-in the configuration dialog. Alternatively, you can use the LRF output format and use the SONY software 
-to transfer the files to the reader. The SONY software pre-paginates the LRF file, 
+in the configuration dialog. Alternatively, you can use the LRF output format and use the SONY software
+to transfer the files to the reader. The SONY software pre-paginates the LRF file,
 thereby reducing the number of resets.
 
 I obtained a recipe for a news site as a .py file from somewhere, how do I use it?
@@ -280,7 +312,7 @@ Take your pick:
 
 Why does |app| show only some of my fonts on OS X?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-|app| embeds fonts in ebook files it creates. E-book files support embedding only TrueType (.ttf) fonts. Most fonts on OS X systems are in .dfont format, thus they cannot be embedded. |app| shows only TrueType fonts founf on your system. You can obtain many TrueType fonts on the web. Simply download the .ttf files and add them to the Library/Fonts directory in your home directory. 
+|app| embeds fonts in ebook files it creates. E-book files support embedding only TrueType (.ttf) fonts. Most fonts on OS X systems are in .dfont format, thus they cannot be embedded. |app| shows only TrueType fonts found on your system. You can obtain many TrueType fonts on the web. Simply download the .ttf files and add them to the Library/Fonts directory in your home directory.
 
 |app| is not starting on Windows?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -303,6 +335,10 @@ Post any output you see in a help message on the `Forum <http://www.mobileread.c
 |app| is not starting on OS X?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+One common cause of failures on OS X is the use of accessibility technologies that are incompatible with the graphics toolkit |app| uses.
+Try turning off VoiceOver if you have it on. Also go to System Preferences->System->Universal Access and turn off the setting for enabling
+access for assistive devices in all the tabs.
+
 You can obtain debug output about why |app| is not starting by running `Console.app`. Debug output will
 be printed to it. If the debug output contains a line that looks like::
 
@@ -312,9 +348,9 @@ then the problem is probably a corrupted font cache. You can clear the cache by
 `instructions <http://www.macworld.com/article/139383/2009/03/fontcacheclear.html>`_. If that doesn't
 solve it, look for a corrupted font file on your system, in ~/Library/Fonts or the like.
 
-
 My antivirus program claims |app| is a virus/trojan?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Your antivirus program is wrong. |app| is a completely open source product. You can actually browse the source code yourself (or hire someone to do it for you) to verify that it is not a virus. Please report the false identification to whatever company you buy your antivirus software from. If the antivirus program is preventing you from downloading/installing |app|, disable it temporarily, install |app| and then re-enable it.
 
 How do I use purchased EPUB books with |app|?
@@ -324,8 +360,8 @@ Most purchased EPUB books have `DRM <http://wiki.mobileread.com/wiki/DRM>`_. Thi
 
 I want some feature added to |app|. What can I do?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-You have two choices: 
- 1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development <http://calibre-ebook.com/get-involved>`_. 
+You have two choices:
+ 1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development <http://calibre-ebook.com/get-involved>`_.
  2. `Open a ticket <http://bugs.calibre-ebook.com/newticket>`_ (you have to register and login first) and hopefully I will find the time to implement your feature.
 
 Can I include |app| on a CD to be distributed with my product/magazine?

src/calibre/manual/resources/titlepage.html

@@ -1,29 +0,0 @@
-<html  xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
-    <head>
-        <title>%(title)s</title>
-        <style type="text/css">
-            body {
-                text-align: center;
-                vertical-align: center;
-                overflow: hidden;
-                font-size: 16pt;
-            }
-            .logo {
-                text-align:center;
-                font-size: 1pt;
-                overflow:hidden;
-            }
-            h1 { font-family: serif; }
-            h2, h4 { font-family: monospace; }
-        </style>
-    </head>
-    <body>
-        <h1>%(title)s</h1>
-        <h4 style="font-family:monospace">%(version)s</h4>
-        <div style="text-align:center">
-            <img class="logo" src="%(img)s" alt="calibre logo" />
-        </div>
-        <h2>%(author)s</h2>
-    </body>
-</html>
-

src/calibre/utils/config.py

@@ -698,6 +698,8 @@ def _prefs():
     # calibre server can execute searches
     c.add_opt('saved_searches', default={}, help=_('List of named saved searches'))
     c.add_opt('user_categories', default={}, help=_('User-created tag browser categories'))
+    c.add_opt('preserve_user_collections', default=True,
+        help=_('Preserve all collections even if not in library metadata.'))
 
     c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.')
     return c

src/calibre/web/feeds/news.py

@@ -585,6 +585,8 @@ def __init__(self, options, log, progress_reporter):
         self.lrf = options.lrf
         self.output_profile = options.output_profile
         self.touchscreen = getattr(self.output_profile, 'touchscreen', False)
+        if self.touchscreen:
+            self.template_css += self.output_profile.touchscreen_news_css
 
         self.output_dir = os.path.abspath(self.output_dir)
         if options.test:
@@ -638,7 +640,8 @@ def __init__(self, options, log, progress_reporter):
         if self.delay > 0:
             self.simultaneous_downloads = 1
 
-        self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else templates.NavBarTemplate()
+        self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else \
+                      templates.NavBarTemplate()
         self.failed_downloads = []
         self.partial_failures = []
 
@@ -726,7 +729,6 @@ def feeds2index(self, feeds):
         timefmt = self.timefmt
         if self.touchscreen:
             templ = templates.TouchscreenIndexTemplate()
-            timefmt = '%A, %d %b %Y'
         return templ.generate(self.title, "mastheadImage.jpg", timefmt, feeds,
                               extra_css=css).render(doctype='xhtml')
 
@@ -752,7 +754,8 @@ def description_limiter(cls, src):
 
 
 
-    def feed2index(self, feed):
+    def feed2index(self, f, feeds):
+        feed = feeds[f]
         if feed.image_url is not None: # Download feed image
             imgdir = os.path.join(self.output_dir, 'images')
             if not os.path.isdir(imgdir):
@@ -782,33 +785,9 @@ def feed2index(self, feed):
         css = self.template_css + '\n\n' +(self.extra_css if self.extra_css else '')
 
         if self.touchscreen:
-            touchscreen_css = u'''
-                    .summary_headline {
-                        font-weight:bold; text-align:left;
-                    }
-
-                    .summary_byline {
-                        text-align:left;
-                        font-family:monospace;
-                    }
-
-                    .summary_text {
-                        text-align:left;
-                    }
-
-                    .feed {
-                        font-family:sans-serif; font-weight:bold; font-size:larger;
-                    }
-
-                    .calibre_navbar {
-                        font-family:monospace;
-                    }
-
-            '''
-
             templ = templates.TouchscreenFeedTemplate()
-            css = touchscreen_css + '\n\n' + (self.extra_css if self.extra_css else '')
-        return templ.generate(feed, self.description_limiter,
+
+        return templ.generate(f, feeds, self.description_limiter,
                               extra_css=css).render(doctype='xhtml')
 
 
@@ -951,7 +930,7 @@ def build_index(self):
         #feeds.restore_duplicates()
 
         for f, feed in enumerate(feeds):
-            html = self.feed2index(feed)
+            html = self.feed2index(f,feeds)
             feed_dir = os.path.join(self.output_dir, 'feed_%d'%f)
             with open(os.path.join(feed_dir, 'index.html'), 'wb') as fi:
                 fi.write(html)

src/calibre/web/feeds/templates.py

@@ -3,9 +3,12 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
+
+import copy
+
 from lxml import html, etree
 from lxml.html.builder import HTML, HEAD, TITLE, STYLE, DIV, BODY, \
-        STRONG, BR, SPAN, A, HR, UL, LI, H2, IMG, P as PT, \
+        STRONG, EM, BR, SPAN, A, HR, UL, LI, H2, H3, IMG, P as PT, \
         TABLE, TD, TR
 
 from calibre import preferred_encoding, strftime, isbytestring
@@ -14,6 +17,7 @@ def CLASS(*args, **kwargs): # class is a reserved word in Python
     kwargs['class'] = ' '.join(args)
     return kwargs
 
+# Regular templates
 class Template(object):
 
     IS_HTML = True
@@ -44,105 +48,35 @@ def render(self, *args, **kwargs):
         return etree.tostring(self.root, encoding='utf-8', xml_declaration=True,
                 pretty_print=True)
 
-class NavBarTemplate(Template):
+class EmbeddedContent(Template):
 
-    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
-                 two_levels, url, __appname__, prefix='', center=True,
-                 extra_css=None, style=None):
-        head = HEAD(TITLE('navbar'))
+    def _generate(self, article, style=None, extra_css=None):
+        content = article.content if article.content else ''
+        summary = article.summary if article.summary else ''
+        text = content if len(content) > len(summary) else summary
+        head = HEAD(TITLE(article.title))
         if style:
             head.append(STYLE(style, type='text/css'))
         if extra_css:
             head.append(STYLE(extra_css, type='text/css'))
 
-        if prefix and not prefix.endswith('/'):
-            prefix += '/'
-        align = 'center' if center else 'left'
-        navbar = DIV(CLASS('calibre_navbar', 'calibre_rescale_70',
-            style='text-align:'+align))
-        if bottom:
-            navbar.append(HR())
-            text = 'This article was downloaded by '
-            p = PT(text, STRONG(__appname__), A(url, href=url), style='text-align:left')
-            p[0].tail = ' from '
-            navbar.append(p)
-            navbar.append(BR())
-            navbar.append(BR())
-        else:
-            next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
-                    else 'article_%d'%(art+1)
-            up = '../..' if art == number_of_articles_in_feed - 1 else '..'
-            href = '%s%s/%s/index.html'%(prefix, up, next)
-            navbar.text = '| '
-            navbar.append(A('Next', href=href))
-        href = '%s../index.html#article_%d'%(prefix, art)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Section Menu', href=href))
-        href = '%s../../index.html#feed_%d'%(prefix, feed)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Main Menu', href=href))
-        if art > 0 and not bottom:
-            href = '%s../article_%d/index.html'%(prefix, art-1)
-            navbar.iterchildren(reversed=True).next().tail = ' | '
-            navbar.append(A('Previous', href=href))
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        if not bottom:
-            navbar.append(HR())
-
-        self.root = HTML(head, BODY(navbar))
-
-class TouchscreenNavBarTemplate(Template):
-
-    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
-                 two_levels, url, __appname__, prefix='', center=True,
-                 extra_css=None, style=None):
-        head = HEAD(TITLE('navbar'))
-        if style:
-            head.append(STYLE(style, type='text/css'))
-        if extra_css:
-            head.append(STYLE(extra_css, type='text/css'))
-
-        if prefix and not prefix.endswith('/'):
-            prefix += '/'
-        align = 'center' if center else 'left'
-        navbar = DIV(CLASS('calibre_navbar', 'calibre_rescale_100',
-            style='text-align:'+align))
-        if bottom:
-            navbar.append(DIV(style="border-top:1px solid gray;border-bottom:1em solid white"))
-            text = 'This article was downloaded by '
-            p = PT(text, STRONG(__appname__), A(url, href=url), style='text-align:left')
-            p[0].tail = ' from '
-            navbar.append(p)
-            navbar.append(BR())
-            navbar.append(BR())
-        else:
-            next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
-                    else 'article_%d'%(art+1)
-            up = '../..' if art == number_of_articles_in_feed - 1 else '..'
-            href = '%s%s/%s/index.html'%(prefix, up, next)
-            navbar.text = '| '
-            navbar.append(A('Next', href=href))
-
-        href = '%s../index.html#article_%d'%(prefix, art)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Section Menu', href=href))
-        href = '%s../../index.html#feed_%d'%(prefix, feed)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Main Menu', href=href))
-        if art > 0 and not bottom:
-            href = '%s../article_%d/index.html'%(prefix, art-1)
-            navbar.iterchildren(reversed=True).next().tail = ' | '
-            navbar.append(A('Previous', href=href))
-
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        if not bottom:
-            navbar.append(DIV(style="border-top:1px solid gray;border-bottom:1em solid white"))
-
-        self.root = HTML(head, BODY(navbar))
+        if isbytestring(text):
+            text = text.decode('utf-8', 'replace')
+        elements = html.fragments_fromstring(text)
+        self.root = HTML(head,
+                BODY(H2(article.title), DIV()))
+        div = self.root.find('body').find('div')
+        if elements and isinstance(elements[0], unicode):
+            div.text = elements[0]
+            elements = list(elements)[1:]
+        for elem in elements:
+            elem.getparent().remove(elem)
+            div.append(elem)
 
 class IndexTemplate(Template):
 
     def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None):
+        self.IS_HTML = False
         if isinstance(datefmt, unicode):
             datefmt = datefmt.encode(preferred_encoding)
         date = strftime(datefmt)
@@ -164,43 +98,10 @@ def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None)
                 CLASS('calibre_rescale_100'))
         self.root = HTML(head, BODY(div))
 
-class TouchscreenIndexTemplate(Template):
-
-    def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None):
-        if isinstance(datefmt, unicode):
-            datefmt = datefmt.encode(preferred_encoding)
-        date = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
-        masthead_p = etree.Element("p")
-        masthead_p.set("style","text-align:center")
-        masthead_img = etree.Element("img")
-        masthead_img.set("src",masthead)
-        masthead_img.set("alt","masthead")
-        masthead_p.append(masthead_img)
-
-        head = HEAD(TITLE(title))
-        if style:
-            head.append(STYLE(style, type='text/css'))
-        if extra_css:
-            head.append(STYLE(extra_css, type='text/css'))
-
-        toc = TABLE(CLASS('toc'),width="100%",border="0",cellpadding="3px")
-        for i, feed in enumerate(feeds):
-            if feed:
-                tr = TR()
-                tr.append(TD( CLASS('calibre_rescale_120'), A(feed.title, href='feed_%d/index.html'%i)))
-                tr.append(TD( '%s' % len(feed.articles), style="text-align:right"))
-                toc.append(tr)
-        div = DIV(
-                masthead_p,
-                PT(date, style='text-align:center'),
-                #DIV(style="border-color:gray;border-top-style:solid;border-width:thin"),
-                DIV(style="border-top:1px solid gray;border-bottom:1em solid white"),
-                toc)
-        self.root = HTML(head, BODY(div))
-
 class FeedTemplate(Template):
 
-    def _generate(self, feed, cutoff, extra_css=None, style=None):
+    def _generate(self, f, feeds, cutoff, extra_css=None, style=None):
+        feed = feeds[f]
         head = HEAD(TITLE(feed.title))
         if style:
             head.append(STYLE(style, type='text/css'))
@@ -248,9 +149,146 @@ def _generate(self, feed, cutoff, extra_css=None, style=None):
 
         self.root = HTML(head, body)
 
+class NavBarTemplate(Template):
+
+    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
+                 two_levels, url, __appname__, prefix='', center=True,
+                 extra_css=None, style=None):
+        head = HEAD(TITLE('navbar'))
+        if style:
+            head.append(STYLE(style, type='text/css'))
+        if extra_css:
+            head.append(STYLE(extra_css, type='text/css'))
+
+        if prefix and not prefix.endswith('/'):
+            prefix += '/'
+        align = 'center' if center else 'left'
+
+        navbar = DIV(CLASS('calibre_navbar', 'calibre_rescale_70',
+            style='text-align:'+align))
+        if bottom:
+            navbar.append(HR())
+            text = 'This article was downloaded by '
+            p = PT(text, STRONG(__appname__), A(url, href=url), style='text-align:left')
+            p[0].tail = ' from '
+            navbar.append(p)
+            navbar.append(BR())
+            navbar.append(BR())
+        else:
+            next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
+                    else 'article_%d'%(art+1)
+            up = '../..' if art == number_of_articles_in_feed - 1 else '..'
+            href = '%s%s/%s/index.html'%(prefix, up, next)
+            navbar.text = '| '
+            navbar.append(A('Next', href=href))
+        href = '%s../index.html#article_%d'%(prefix, art)
+        navbar.iterchildren(reversed=True).next().tail = ' | '
+        navbar.append(A('Section Menu', href=href))
+        href = '%s../../index.html#feed_%d'%(prefix, feed)
+        navbar.iterchildren(reversed=True).next().tail = ' | '
+        navbar.append(A('Main Menu', href=href))
+        if art > 0 and not bottom:
+            href = '%s../article_%d/index.html'%(prefix, art-1)
+            navbar.iterchildren(reversed=True).next().tail = ' | '
+            navbar.append(A('Previous', href=href))
+        navbar.iterchildren(reversed=True).next().tail = ' | '
+        if not bottom:
+            navbar.append(HR())
+
+        self.root = HTML(head, BODY(navbar))
+
+
+# Touchscreen templates
+class TouchscreenIndexTemplate(Template):
+
+    def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None):
+
+        self.IS_HTML = False
+
+        if isinstance(datefmt, unicode):
+            datefmt = datefmt.encode(preferred_encoding)
+        date = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
+        masthead_p = etree.Element("p")
+        masthead_p.set("style","text-align:center")
+        masthead_img = etree.Element("img")
+        masthead_img.set("src",masthead)
+        masthead_img.set("alt","masthead")
+        masthead_p.append(masthead_img)
+
+        head = HEAD(TITLE(title))
+        if style:
+            head.append(STYLE(style, type='text/css'))
+        if extra_css:
+            head.append(STYLE(extra_css, type='text/css'))
+
+        toc = TABLE(CLASS('toc'),width="100%",border="0",cellpadding="3px")
+        for i, feed in enumerate(feeds):
+            if feed:
+                tr = TR()
+                tr.append(TD( CLASS('calibre_rescale_120'), A(feed.title, href='feed_%d/index.html'%i)))
+                tr.append(TD( '%s' % len(feed.articles), style="text-align:right"))
+                toc.append(tr)
+        div = DIV(
+                masthead_p,
+                H3(CLASS('publish_date'),date),
+                DIV(CLASS('divider')),
+                toc)
+        self.root = HTML(head, BODY(div))
+
 class TouchscreenFeedTemplate(Template):
 
-    def _generate(self, feed, cutoff, extra_css=None, style=None):
+    def _generate(self, f, feeds, cutoff, extra_css=None, style=None):
+
+        def trim_title(title,clip=18):
+            if len(title)>clip:
+                tokens = title.split(' ')
+                new_title_tokens = []
+                new_title_len = 0
+                if len(tokens[0]) > clip:
+                    return tokens[0][:clip] + '...'
+                for token in tokens:
+                    if len(token) + new_title_len < clip:
+                        new_title_tokens.append(token)
+                        new_title_len += len(token)
+                    else:
+                        new_title_tokens.append('...')
+                        title = ' '.join(new_title_tokens)
+                        break
+            return title
+
+        self.IS_HTML = False
+        feed = feeds[f]
+
+        # Construct the navbar
+        navbar_t = TABLE(CLASS('touchscreen_navbar'))
+        navbar_tr = TR()
+
+        # Previous Section
+        link = ''
+        if f > 0:
+            link = A(CLASS('feed_link'),
+                     trim_title(feeds[f-1].title),
+                     href = '../feed_%d/index.html' % int(f-1))
+        navbar_tr.append(TD(link, width="40%", align="center"))
+
+        # Up to Sections
+        link = A(STRONG('Sections'), href="../index.html")
+        navbar_tr.append(TD(link,width="20%",align="center"))
+
+        # Next Section
+        link = ''
+        if f < len(feeds)-1:
+            link = A(CLASS('feed_link'),
+                     trim_title(feeds[f+1].title),
+                     href = '../feed_%d/index.html' % int(f+1))
+        navbar_tr.append(TD(link, width="40%", align="center", ))
+        navbar_t.append(navbar_tr)
+        top_navbar = navbar_t
+        bottom_navbar = copy.copy(navbar_t)
+        #print "\n%s\n" % etree.tostring(navbar_t, pretty_print=True)
+
+
+        # Build the page
         head = HEAD(TITLE(feed.title))
         if style:
             head.append(STYLE(style, type='text/css'))
@@ -258,10 +296,11 @@ def _generate(self, feed, cutoff, extra_css=None, style=None):
             head.append(STYLE(extra_css, type='text/css'))
         body = BODY(style='page-break-before:always')
         div = DIV(
-                H2(feed.title, CLASS('calibre_feed_title', 'calibre_rescale_160')),
-                DIV(style="border-top:1px solid gray;border-bottom:1em solid white")
+                top_navbar,
+                H2(feed.title, CLASS('feed_title'))
                 )
         body.append(div)
+
         if getattr(feed, 'image', None):
             div.append(DIV(IMG(
                 alt = feed.image_alt if feed.image_alt else '',
@@ -280,65 +319,64 @@ def _generate(self, feed, cutoff, extra_css=None, style=None):
                 continue
             tr = TR()
 
-            if True:
-                div_td = DIV(
-                        A(article.title, CLASS('summary_headline','calibre_rescale_120',
-                                        href=article.url)),
-                        style="display:inline-block")
-                if article.author:
-                    div_td.append(DIV(article.author,
-                        CLASS('summary_byline', 'calibre_rescale_100')))
-                if article.summary:
-                    div_td.append(DIV(cutoff(article.text_summary),
-                        CLASS('summary_text', 'calibre_rescale_100')))
-                tr.append(TD(div_td))
-            else:
-                td = TD(
-                        A(article.title, CLASS('summary_headline','calibre_rescale_120',
-                                        href=article.url))
-                        )
-                if article.author:
-                    td.append(DIV(article.author,
-                        CLASS('summary_byline', 'calibre_rescale_100')))
-                if article.summary:
-                    td.append(DIV(cutoff(article.text_summary),
-                        CLASS('summary_text', 'calibre_rescale_100')))
-
-                tr.append(td)
-
+            div_td = DIV(
+                    A(article.title, CLASS('summary_headline','calibre_rescale_120',
+                                    href=article.url)),
+                    style="display:inline-block")
+            if article.author:
+                div_td.append(DIV(article.author,
+                    CLASS('summary_byline', 'calibre_rescale_100')))
+            if article.summary:
+                div_td.append(DIV(cutoff(article.text_summary),
+                    CLASS('summary_text', 'calibre_rescale_100')))
+            tr.append(TD(div_td))
             toc.append(tr)
+
         div.append(toc)
-
-        navbar = DIV('| ', CLASS('calibre_navbar', 'calibre_rescale_100'),style='text-align:center')
-        link = A('Up one level', href="../index.html")
-        link.tail = ' |'
-        navbar.append(link)
-        div.append(navbar)
-
+        div.append(BR())
+        div.append(bottom_navbar)
         self.root = HTML(head, body)
 
-class EmbeddedContent(Template):
+class TouchscreenNavBarTemplate(Template):
 
-    def _generate(self, article, style=None, extra_css=None):
-        content = article.content if article.content else ''
-        summary = article.summary if article.summary else ''
-        text = content if len(content) > len(summary) else summary
-        head = HEAD(TITLE(article.title))
+    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
+                 two_levels, url, __appname__, prefix='', center=True,
+                 extra_css=None, style=None):
+        head = HEAD(TITLE('navbar'))
         if style:
             head.append(STYLE(style, type='text/css'))
         if extra_css:
             head.append(STYLE(extra_css, type='text/css'))
 
-        if isbytestring(text):
-            text = text.decode('utf-8', 'replace')
-        elements = html.fragments_fromstring(text)
-        self.root = HTML(head,
-                BODY(H2(article.title), DIV()))
-        div = self.root.find('body').find('div')
-        if elements and isinstance(elements[0], unicode):
-            div.text = elements[0]
-            elements = list(elements)[1:]
-        for elem in elements:
-            elem.getparent().remove(elem)
-            div.append(elem)
+        navbar = DIV()
+        navbar_t = TABLE(CLASS('touchscreen_navbar'))
+        navbar_tr = TR()
 
+        # | Previous
+        if art > 0:
+            href = '%s../article_%d/index.html'%(prefix, art-1)
+            navbar_tr.append(TD(A(EM('Previous'),href=href),
+                               width="32%"))
+        else:
+            navbar_tr.append(TD('', width="32%"))
+
+        # | Articles | Sections |
+        href = '%s../index.html#article_%d'%(prefix, art)
+        navbar_tr.append(TD(A(STRONG('Articles'), href=href),width="18%"))
+
+        href = '%s../../index.html#feed_%d'%(prefix, feed)
+        navbar_tr.append(TD(A(STRONG('Sections'), href=href),width="18%"))
+
+        # | Next
+        next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
+                else 'article_%d'%(art+1)
+        up = '../..' if art == number_of_articles_in_feed - 1 else '..'
+        href = '%s%s/%s/index.html'%(prefix, up, next)
+
+        navbar_tr.append(TD(A(EM('Next'),href=href),
+                           width="32%"))
+        navbar_t.append(navbar_tr)
+        navbar.append(navbar_t)
+        #print "\n%s\n" % etree.tostring(navbar, pretty_print=True)
+
+        self.root = HTML(head, BODY(navbar))