KG updates

This commit is contained in:
GRiker 2011-06-23 10:12:58 -06:00
commit 9bc3676dbc
18 changed files with 1126 additions and 642 deletions

View file

@ -261,7 +261,7 @@ class OutputFormatPlugin(Plugin):
@property
def description(self):
return _('Convert ebooks to the %s format'%self.file_type)
return _('Convert ebooks to the %s format')%self.file_type
def __init__(self, *args):
Plugin.__init__(self, *args)

View file

@ -125,9 +125,9 @@ def update_booklist(prefix, path, title, authors, mime, date, ContentType, Image
# this shows an expired Collection so the user can decide to delete the book
if expired == 3:
playlist_map[lpath].append('Expired')
# Favourites are supported on the touch but the data field is there on most earlier models
# A SHORTLIST is supported on the touch but the data field is there on most earlier models
if favouritesindex == 1:
playlist_map[lpath].append('Favourite')
playlist_map[lpath].append('Shortlist')
path = self.normalize_path(path)
# print "Normalized FileName: " + path
@ -557,6 +557,7 @@ def update_device_database_collections(self, booklists, collections_attributes,
if collections:
# Process any collections that exist
for category, books in collections.items():
# debug_print (category)
if category == 'Im_Reading':
# Reset Im_Reading list in the database
if oncard == 'carda':
@ -575,7 +576,8 @@ def update_device_database_collections(self, booklists, collections_attributes,
for book in books:
# debug_print('Title:', book.title, 'lpath:', book.path)
book.device_collections = ['Im_Reading']
if 'Im_Reading' not in book.device_collections:
book.device_collections.append('Im_Reading')
extension = os.path.splitext(book.path)[1]
ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
@ -618,7 +620,8 @@ def update_device_database_collections(self, booklists, collections_attributes,
for book in books:
# debug_print('Title:', book.title, 'lpath:', book.path)
book.device_collections = ['Read']
if 'Read' not in book.device_collections:
book.device_collections.append('Read')
extension = os.path.splitext(book.path)[1]
ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
@ -654,7 +657,8 @@ def update_device_database_collections(self, booklists, collections_attributes,
for book in books:
# debug_print('Title:', book.title, 'lpath:', book.path)
book.device_collections = ['Closed']
if 'Closed' not in book.device_collections:
book.device_collections.append('Closed')
extension = os.path.splitext(book.path)[1]
ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
@ -672,6 +676,44 @@ def update_device_database_collections(self, booklists, collections_attributes,
else:
connection.commit()
# debug_print('Database: Commit set ReadStatus as Closed')
if category == 'Shortlist':
# Reset FavouritesIndex list in the database
if oncard == 'carda':
query= 'update content set FavouritesIndex=-1 where BookID is Null and ContentID like \'file:///mnt/sd/%\''
elif oncard != 'carda' and oncard != 'cardb':
query= 'update content set FavouritesIndex=-1 where BookID is Null and ContentID not like \'file:///mnt/sd/%\''
try:
cursor.execute (query)
except:
debug_print('Database Exception: Unable to reset Shortlist list')
raise
else:
# debug_print('Commit: Reset Shortlist list')
connection.commit()
for book in books:
# debug_print('Title:', book.title, 'lpath:', book.path)
if 'Shortlist' not in book.device_collections:
book.device_collections.append('Shortlist')
# debug_print ("Shortlist found for: ", book.title)
extension = os.path.splitext(book.path)[1]
ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
ContentID = self.contentid_from_path(book.path, ContentType)
# datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())
t = (ContentID,)
try:
cursor.execute('update content set FavouritesIndex=1 where BookID is Null and ContentID = ?', t)
except:
debug_print('Database Exception: Unable set book as Shortlist')
raise
else:
connection.commit()
# debug_print('Database: Commit set Shortlist as Shortlist')
else: # No collections
# Since no collections exist the ReadStatus needs to be reset to 0 (Unread)
print "Reseting ReadStatus to 0"

View file

@ -19,8 +19,9 @@ class TECLAST_K3(USBMS):
PRODUCT_ID = [0x3203]
BCD = [0x0000, 0x0100]
VENDOR_NAME = 'TECLAST'
WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['DIGITAL_PLAYER', 'TL-K5']
VENDOR_NAME = ['TECLAST', 'IMAGIN']
WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['DIGITAL_PLAYER', 'TL-K5',
'EREADER']
MAIN_MEMORY_VOLUME_LABEL = 'K3 Main Memory'
STORAGE_CARD_VOLUME_LABEL = 'K3 Storage Card'

View file

@ -13,8 +13,7 @@
class ShowQuickviewAction(InterfaceAction):
name = 'Show quickview'
action_spec = (_('Show quickview'), 'user_profile.png', None,
_('Q'))
action_spec = (_('Show quickview'), 'search.png', None, _('Q'))
dont_add_to = frozenset(['menubar-device', 'toolbar-device', 'context-menu-device'])
action_type = 'current'

View file

@ -451,7 +451,8 @@ def __init__(self, parent, db, callback, rows, path, opts,
self.callback_called = False
self.rq = Queue()
self.ids = [x for x in map(db.id, [r.row() for r in rows]) if x is not None]
self.pd.set_max(len(self.ids))
self.pd_max = len(self.ids)
self.pd.set_max(0)
self.pd.value = 0
self.failures = set([])
@ -510,6 +511,8 @@ def get_result(self):
id, title, ok, tb = self.rq.get_nowait()
except Empty:
return
if self.pd.max != self.pd_max:
self.pd.max = self.pd_max
self.pd.value += 1
self.ids.remove(id)
if not isinstance(title, unicode):

View file

@ -25,7 +25,7 @@ class Base(object):
def __init__(self, db, col_id, parent=None):
self.db, self.col_id = db, col_id
self.col_metadata = db.custom_column_num_map[col_id]
self.initial_val = None
self.initial_val = self.widgets = None
self.setup_ui(parent)
def initialize(self, book_id):
@ -54,6 +54,9 @@ def normalize_db_val(self, val):
def normalize_ui_val(self, val):
return val
def break_cycles(self):
self.db = self.widgets = self.initial_val = None
class Bool(Base):
def setup_ui(self, parent):

View file

@ -53,6 +53,13 @@ def set_min(self, min):
def set_max(self, max):
self.bar.setMaximum(max)
@dynamic_property
def max(self):
def fget(self): return self.bar.maximum()
def fset(self, val): self.bar.setMaximum(val)
return property(fget=fget, fset=fset)
def _canceled(self, *args):
self.canceled = True
self.button_box.setDisabled(True)

View file

@ -5,23 +5,27 @@
from PyQt4.Qt import (Qt, QDialog, QAbstractItemView, QTableWidgetItem,
QListWidgetItem, QByteArray, QModelIndex)
QListWidgetItem, QByteArray, QModelIndex, QCoreApplication)
from calibre.gui2.dialogs.quickview_ui import Ui_Quickview
from calibre.utils.icu import sort_key
from calibre.gui2 import gprefs
class tableItem(QTableWidgetItem):
class TableItem(QTableWidgetItem):
'''
A QTableWidgetItem that sorts on a separate string and uses ICU rules
'''
def __init__(self, val):
def __init__(self, val, sort):
self.sort = sort
QTableWidgetItem.__init__(self, val)
self.setFlags(Qt.ItemIsEnabled|Qt.ItemIsSelectable)
def __ge__(self, other):
return sort_key(unicode(self.text())) >= sort_key(unicode(other.text()))
return sort_key(self.sort) >= sort_key(other.sort)
def __lt__(self, other):
return sort_key(unicode(self.text())) < sort_key(unicode(other.text()))
return sort_key(self.sort) < sort_key(other.sort)
class Quickview(QDialog, Ui_Quickview):
@ -31,12 +35,16 @@ def __init__(self, gui, view, row):
self.setupUi(self)
self.isClosed = False
self.books_table_column_widths = None
try:
self.books_table_column_widths = \
gprefs.get('quickview_dialog_books_table_widths', None)
geom = gprefs.get('quickview_dialog_geometry', bytearray(''))
self.restoreGeometry(QByteArray(geom))
except:
pass
# Remove the help button from the window title bar
icon = self.windowIcon()
self.setWindowFlags(self.windowFlags()&(~Qt.WindowContextHelpButtonHint))
self.setWindowIcon(icon)
@ -44,11 +52,16 @@ def __init__(self, gui, view, row):
self.db = view.model().db
self.view = view
self.gui = gui
self.is_closed = False
self.current_book_id = None
self.current_key = None
self.use_current_key_for_next_refresh = False
self.last_search = None
self.items.setSelectionMode(QAbstractItemView.SingleSelection)
self.items.currentTextChanged.connect(self.item_selected)
# self.items.setFixedWidth(150)
# Set up the books table columns
self.books_table.setSelectionBehavior(QAbstractItemView.SelectRows)
self.books_table.setSelectionMode(QAbstractItemView.SingleSelection)
self.books_table.setColumnCount(3)
@ -60,67 +73,81 @@ def __init__(self, gui, view, row):
self.books_table.setHorizontalHeaderItem(2, t)
self.books_table_header_height = self.books_table.height()
self.books_table.cellDoubleClicked.connect(self.book_doubleclicked)
self.books_table.sortByColumn(0, Qt.AscendingOrder)
self.is_closed = False
self.current_book_id = None
self.current_key = None
self.use_current_key_for_next_refresh = False
self.last_search = None
# get the standard table row height. Do this here because calling
# resizeRowsToContents can word wrap long cell contents, creating
# double-high rows
self.books_table.setRowCount(1)
self.books_table.setItem(0, 0, TableItem('A', ''))
self.books_table.resizeRowsToContents()
self.books_table_row_height = self.books_table.rowHeight(0)
self.books_table.setRowCount(0)
# Add the data
self.refresh(row)
# self.connect(self.view.selectionModel(), SIGNAL('currentChanged(QModelIndex,QModelIndex)'), self.slave)
self.view.selectionModel().currentChanged[QModelIndex,QModelIndex].connect(self.slave)
QCoreApplication.instance().aboutToQuit.connect(self.save_state)
self.search_button.clicked.connect(self.do_search)
# search button
def do_search(self):
if self.last_search is not None:
self.use_current_key_for_next_refresh = True
self.gui.search.set_search_string(self.last_search)
# clicks on the items listWidget
def item_selected(self, txt):
self.fill_in_books_box(unicode(txt))
# Given a cell in the library view, display the information
def refresh(self, idx):
bv_row = idx.row()
key = self.view.model().column_map[idx.column()]
book_id = self.view.model().id(bv_row)
# Double-clicking on a book to show it in the library view will result
# in a signal emitted for column 1 of the book row. Use the original
# column for this signal.
if self.use_current_key_for_next_refresh:
key = self.current_key
self.use_current_key_for_next_refresh = False
else:
# Only show items for categories
if not self.db.field_metadata[key]['is_category']:
if self.current_key is None:
return
key = self.current_key
self.items_label.setText('{0} ({1})'.format(
self.db.field_metadata[key]['name'], key))
self.items.blockSignals(True)
self.items.clear()
self.books_table.setRowCount(0)
mi = self.db.get_metadata(book_id, index_is_id=True, get_user_categories=False)
vals = mi.get(key, None)
if not vals:
return
if not isinstance(vals, list):
vals = [vals]
vals.sort(key=sort_key)
if vals:
if not isinstance(vals, list):
vals = [vals]
vals.sort(key=sort_key)
self.items.blockSignals(True)
for v in vals:
a = QListWidgetItem(v)
self.items.addItem(a)
self.items.setCurrentRow(0)
for v in vals:
a = QListWidgetItem(v)
self.items.addItem(a)
self.items.setCurrentRow(0)
self.current_book_id = book_id
self.current_key = key
self.fill_in_books_box(vals[0])
self.items.blockSignals(False)
self.current_book_id = book_id
self.current_key = key
self.fill_in_books_box(vals[0])
def fill_in_books_box(self, selected_item):
# Do a bit of fix-up on the items so that the search works.
if selected_item.startswith('.'):
sv = '.' + selected_item
else:
@ -129,43 +156,82 @@ def fill_in_books_box(self, selected_item):
self.last_search = self.current_key+':"=' + sv + '"'
books = self.db.search_getting_ids(self.last_search,
self.db.data.search_restriction)
self.books_table.setRowCount(len(books))
self.books_label.setText(_('Books with selected item: {0}').format(len(books)))
select_row = None
select_item = None
self.books_table.setSortingEnabled(False)
for row, b in enumerate(books):
mi = self.db.get_metadata(b, index_is_id=True, get_user_categories=False)
a = tableItem(mi.title)
a = TableItem(mi.title, mi.title_sort)
a.setData(Qt.UserRole, b)
self.books_table.setItem(row, 0, a)
a = tableItem(' & '.join(mi.authors))
if b == self.current_book_id:
select_item = a
a = TableItem(' & '.join(mi.authors), mi.author_sort)
self.books_table.setItem(row, 1, a)
series = mi.format_field('series')[1]
if series is None:
series = ''
a = tableItem(series)
a = TableItem(series, series)
self.books_table.setItem(row, 2, a)
if b == self.current_book_id:
select_row = row
self.books_table.setRowHeight(row, self.books_table_row_height)
self.books_table.resizeColumnsToContents()
# self.books_table.resizeRowsToContents()
if select_row is not None:
self.books_table.selectRow(select_row)
self.books_table.setSortingEnabled(True)
if select_item is not None:
self.books_table.setCurrentItem(select_item)
self.books_table.scrollToItem(select_item, QAbstractItemView.PositionAtCenter)
# Deal with sizing the table columns. Done here because the numbers are not
# correct until the first paint.
def resizeEvent(self, *args):
QDialog.resizeEvent(self, *args)
if self.books_table_column_widths is not None:
for c,w in enumerate(self.books_table_column_widths):
self.books_table.setColumnWidth(c, w)
else:
# the vertical scroll bar might not be rendered, so might not yet
# have a width. Assume 25. Not a problem because user-changed column
# widths will be remembered
w = self.books_table.width() - 25 - self.books_table.verticalHeader().width()
w /= self.books_table.columnCount()
for c in range(0, self.books_table.columnCount()):
self.books_table.setColumnWidth(c, w)
self.save_state()
def book_doubleclicked(self, row, column):
self.use_current_key_for_next_refresh = True
self.view.select_rows([self.books_table.item(row, 0).data(Qt.UserRole).toInt()[0]])
# called when a book is clicked on the library view
def slave(self, current, previous):
if self.is_closed:
return
self.refresh(current)
self.view.activateWindow()
def done(self, r):
geom = bytearray(self.saveGeometry())
gprefs['quickview_dialog_geometry'] = geom
def save_state(self):
if self.is_closed:
return
self.books_table_column_widths = []
for c in range(0, self.books_table.columnCount()):
self.books_table_column_widths.append(self.books_table.columnWidth(c))
gprefs['quickview_dialog_books_table_widths'] = self.books_table_column_widths
gprefs['quickview_dialog_geometry'] = bytearray(self.saveGeometry())
def close(self):
self.save_state()
# clean up to prevent memory leaks
self.db = self.view = self.gui = None
self.is_closed = True
QDialog.done(self, r)
# called by the window system
def closeEvent(self, *args):
self.close()
QDialog.closeEvent(self, *args)
# called by the close button
def reject(self):
self.close()
QDialog.reject(self)

View file

@ -21,9 +21,10 @@
from calibre.ebooks.metadata import (title_sort, authors_to_string,
string_to_authors, check_isbn, authors_to_sort_string)
from calibre.ebooks.metadata.meta import get_metadata
from calibre.gui2 import (file_icon_provider, UNDEFINED_QDATE, UNDEFINED_DATE,
from calibre.gui2 import (file_icon_provider, UNDEFINED_QDATE,
choose_files, error_dialog, choose_images)
from calibre.utils.date import local_tz, qt_to_dt
from calibre.utils.date import (local_tz, qt_to_dt, as_local_time,
UNDEFINED_DATE)
from calibre import strftime
from calibre.ebooks import BOOK_EXTENSIONS
from calibre.customize.ui import run_plugins_on_import
@ -125,6 +126,9 @@ def fset(self, val):
return property(fget=fget, fset=fset)
def break_cycles(self):
self.dialog = None
class TitleSortEdit(TitleEdit):
TITLE_ATTR = 'title_sort'
@ -150,6 +154,7 @@ def __init__(self, parent, title_edit, autogen_button):
self.title_edit.textChanged.connect(self.update_state)
self.textChanged.connect(self.update_state)
self.autogen_button = autogen_button
autogen_button.clicked.connect(self.auto_generate)
self.update_state()
@ -168,6 +173,9 @@ def update_state(self, *args):
def auto_generate(self, *args):
self.current_val = title_sort(self.title_edit.current_val)
self.title_edit.textChanged.disconnect()
self.textChanged.disconnect()
self.autogen_button.clicked.disconnect()
# }}}
@ -185,6 +193,7 @@ def __init__(self, parent, manage_authors):
self.setWhatsThis(self.TOOLTIP)
self.setEditable(True)
self.setSizeAdjustPolicy(self.AdjustToMinimumContentsLengthWithIcon)
self.manage_authors_signal = manage_authors
manage_authors.triggered.connect(self.manage_authors)
def manage_authors(self):
@ -269,6 +278,10 @@ def fset(self, val):
return property(fget=fget, fset=fset)
def break_cycles(self):
self.db = self.dialog = None
self.manage_authors_signal.triggered.disconnect()
class AuthorSortEdit(EnLineEdit):
TOOLTIP = _('Specify how the author(s) of this book should be sorted. '
@ -297,6 +310,10 @@ def __init__(self, parent, authors_edit, autogen_button, db,
self.authors_edit.editTextChanged.connect(self.update_state_and_val)
self.textChanged.connect(self.update_state)
self.autogen_button = autogen_button
self.copy_a_to_as_action = copy_a_to_as_action
self.copy_as_to_a_action = copy_as_to_a_action
autogen_button.clicked.connect(self.auto_generate)
copy_a_to_as_action.triggered.connect(self.auto_generate)
copy_as_to_a_action.triggered.connect(self.copy_to_authors)
@ -368,6 +385,15 @@ def commit(self, db, id_):
db.set_author_sort(id_, aus, notify=False, commit=False)
return True
def break_cycles(self):
self.db = None
self.authors_edit.editTextChanged.disconnect()
self.textChanged.disconnect()
self.autogen_button.clicked.disconnect()
self.copy_a_to_as_action.triggered.disconnect()
self.copy_as_to_a_action.triggered.disconnect()
self.authors_edit = None
# }}}
# Series {{{
@ -427,6 +453,10 @@ def commit(self, db, id_):
commit=True, allow_case_change=True)
return True
def break_cycles(self):
self.dialog = None
class SeriesIndexEdit(QDoubleSpinBox):
TOOLTIP = ''
@ -488,6 +518,11 @@ def increment(self):
import traceback
traceback.print_exc()
def break_cycles(self):
self.series_edit.currentIndexChanged.disconnect()
self.series_edit.editTextChanged.disconnect()
self.series_edit.lineEdit().editingFinished.disconnect()
self.db = self.series_edit = self.dialog = None
# }}}
@ -699,6 +734,8 @@ def get_selected_format_metadata(self, db, id_):
if old != prefs['read_file_metadata']:
prefs['read_file_metadata'] = old
def break_cycles(self):
self.dialog = None
# }}}
class Cover(ImageView): # {{{
@ -860,6 +897,10 @@ def commit(self, db, id_):
db.remove_cover(id_, notify=False, commit=False)
return True
def break_cycles(self):
self.cover_changed.disconnect()
self.dialog = self._cdata = self.current_val = self.original_val = None
# }}}
class CommentsEdit(Editor): # {{{
@ -1211,6 +1252,7 @@ def fget(self):
def fset(self, val):
if val is None:
val = UNDEFINED_DATE
val = as_local_time(val)
self.setDate(QDate(val.year, val.month, val.day))
return property(fget=fget, fset=fset)

View file

@ -481,6 +481,13 @@ def disconnect(signal):
x = getattr(self, b, None)
if x is not None:
disconnect(x.clicked)
for widget in self.basic_metadata_widgets:
bc = getattr(widget, 'break_cycles', None)
if bc is not None and callable(bc):
bc()
for widget in getattr(self, 'custom_metadata_widgets', []):
widget.break_cycles()
# }}}
class Splitter(QSplitter):

View file

@ -72,19 +72,27 @@ def __init__(self, fm, parent=None):
self.l = l = QGridLayout(self)
self.setLayout(l)
self.l1 = l1 = QLabel(_('If the '))
texts = _('If the ___ column ___ values')
try:
one, two, three = texts.split('___')
except:
one, two, three = 'If the ', ' column ', ' value '
self.l1 = l1 = QLabel(one)
l.addWidget(l1, 0, 0)
self.column_box = QComboBox(self)
l.addWidget(self.column_box, 0, 1)
self.l2 = l2 = QLabel(_(' column '))
self.l2 = l2 = QLabel(two)
l.addWidget(l2, 0, 2)
self.action_box = QComboBox(self)
l.addWidget(self.action_box, 0, 3)
self.l3 = l3 = QLabel(_(' value '))
self.l3 = l3 = QLabel(three)
l.addWidget(l3, 0, 4)
self.value_box = QLineEdit(self)

View file

@ -10,6 +10,7 @@
from calibre.gui2.preferences.misc_ui import Ui_Form
from calibre.gui2 import error_dialog, config, open_local_file, info_dialog
from calibre.constants import isosx
from calibre import get_proxies
class WorkersSetting(Setting):
@ -33,6 +34,13 @@ def genesis(self, gui):
self.user_defined_device_button.clicked.connect(self.user_defined_device)
self.button_osx_symlinks.clicked.connect(self.create_symlinks)
self.button_osx_symlinks.setVisible(isosx)
proxies = get_proxies(debug=False)
txt = _('No proxies used')
if proxies:
lines = ['<br><code>%s: %s</code>'%(t, p) for t, p in
proxies.iteritems()]
txt = _('<b>Using proxies:</b>') + ''.join(lines)
self.proxies.setText(txt)
def debug_device_detection(self, *args):
from calibre.gui2.preferences.device_debug import DebugDevice

View file

@ -118,7 +118,7 @@
</property>
</widget>
</item>
<item row="20" column="0">
<item row="21" column="0">
<spacer name="verticalSpacer_9">
<property name="orientation">
<enum>Qt::Vertical</enum>
@ -131,6 +131,13 @@
</property>
</spacer>
</item>
<item row="10" column="0" colspan="2">
<widget class="QLabel" name="proxies">
<property name="text">
<string/>
</property>
</widget>
</item>
</layout>
</widget>
<resources/>

View file

@ -627,7 +627,8 @@ def break_cycles(self):
except:
pass
self.parent = self.icon_state_map = self.bold_font = self.tag = \
self.icon = self.children = None
self.icon = self.children = self.tooltip = \
self.py_name = self.id_set = self.category_key = None
def __str__(self):
if self.type == self.ROOT:
@ -1121,7 +1122,7 @@ def set_search_restriction(self, s):
self.search_restriction = s
def get_node_tree(self, sort):
old_row_map = self.row_map[:]
old_row_map_len = len(self.row_map)
self.row_map = []
self.categories = {}
@ -1176,7 +1177,7 @@ def get_node_tree(self, sort):
self.row_map.append(category)
self.categories[category] = tb_categories[category]['name']
if len(old_row_map) != 0 and len(old_row_map) != len(self.row_map):
if old_row_map_len != 0 and old_row_map_len != len(self.row_map):
# A category has been added or removed. We must force a rebuild of
# the model
return None
@ -1367,6 +1368,9 @@ def process_one_node(category, state_map): # {{{
self.beginRemoveRows(self.createIndex(category.row(), 0, category),
start, len(child_map)-1)
category.children = ctags
for i in range(start, len(child_map)):
child_map[i].break_cycles()
child_map = None
self.endRemoveRows()
else:
state_map = {}

View file

@ -7,7 +7,7 @@
The database used to store ebook metadata
'''
import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
json, uuid, tempfile
json, uuid, tempfile, hashlib
import threading, random
from itertools import repeat
from math import ceil
@ -1118,9 +1118,31 @@ def has_format(self, index, format, index_is_id=False):
return self.format_abspath(index, format, index_is_id) is not None
def format_last_modified(self, id_, fmt):
m = self.format_metadata(id_, fmt)
if m:
return m['mtime']
def format_metadata(self, id_, fmt):
path = self.format_abspath(id_, fmt, index_is_id=True)
ans = {}
if path is not None:
return utcfromtimestamp(os.stat(path).st_mtime)
stat = os.stat(path)
ans['size'] = stat.st_size
ans['mtime'] = utcfromtimestamp(stat.st_mtime)
return ans
def format_hash(self, id_, fmt):
path = self.format_abspath(id_, fmt, index_is_id=True)
if path is None:
raise NoSuchFormat('Record %d has no fmt: %s'%(id_, fmt))
sha = hashlib.sha256()
with lopen(path, 'rb') as f:
while True:
raw = f.read(SPOOL_SIZE)
sha.update(raw)
if len(raw) < SPOOL_SIZE:
break
return sha.hexdigest()
def format_abspath(self, index, format, index_is_id=False):
'''

File diff suppressed because it is too large Load diff

View file

@ -123,6 +123,14 @@ def isoformat(date_time, assume_utc=False, as_utc=True, sep='T'):
date_time = date_time.astimezone(_utc_tz if as_utc else _local_tz)
return unicode(date_time.isoformat(sep))
def as_local_time(date_time, assume_utc=True):
if not hasattr(date_time, 'tzinfo'):
return date_time
if date_time.tzinfo is None:
date_time = date_time.replace(tzinfo=_utc_tz if assume_utc else
_local_tz)
return date_time.astimezone(_local_tz)
def now():
return datetime.now().replace(tzinfo=_local_tz)

View file

@ -8,61 +8,157 @@
'''
Measure memory usage of the current process.
The key function is memory() which returns the current memory usage in bytes.
The key function is memory() which returns the current memory usage in MB.
You can pass a number to memory and it will be subtracted from the returned
value.
'''
import gc, os
import gc, os, re
from calibre.constants import iswindows, islinux
if islinux:
## {{{ http://code.activestate.com/recipes/286222/ (r1)
# Taken, with thanks, from:
# http://wingolog.org/archives/2007/11/27/reducing-the-footprint-of-python-applications
_proc_status = '/proc/%d/status' % os.getpid()
def permute(args):
ret = []
if args:
first = args.pop(0)
for y in permute(args):
for x in first:
ret.append(x + y)
else:
ret.append('')
return ret
_scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
'KB': 1024.0, 'MB': 1024.0*1024.0}
def parsed_groups(match, *types):
groups = match.groups()
assert len(groups) == len(types)
return tuple([type(group) for group, type in zip(groups, types)])
def _VmB(VmKey):
'''Private.
'''
global _proc_status, _scale
# get pseudo file /proc/<pid>/status
try:
t = open(_proc_status)
v = t.read()
t.close()
except:
return 0.0 # non-Linux?
# get VmKey line e.g. 'VmRSS: 9999 kB\n ...'
i = v.index(VmKey)
v = v[i:].split(None, 3) # whitespace
if len(v) < 3:
return 0.0 # invalid format?
# convert Vm value to bytes
return float(v[1]) * _scale[v[2]]
class VMA(dict):
def __init__(self, *args):
(self.start, self.end, self.perms, self.offset,
self.major, self.minor, self.inode, self.filename) = args
def parse_smaps(pid):
with open('/proc/%s/smaps'%pid, 'r') as maps:
hex = lambda s: int(s, 16)
ret = []
header = re.compile(r'^([0-9a-f]+)-([0-9a-f]+) (....) ([0-9a-f]+) '
r'(..):(..) (\d+) *(.*)$')
detail = re.compile(r'^(.*): +(\d+) kB')
for line in maps:
m = header.match(line)
if m:
vma = VMA(*parsed_groups(m, hex, hex, str, hex, str, str, int, str))
ret.append(vma)
else:
m = detail.match(line)
if m:
k, v = parsed_groups(m, str, int)
assert k not in vma
vma[k] = v
else:
print 'unparseable line:', line
return ret
perms = permute(['r-', 'w-', 'x-', 'ps'])
def make_summary_dicts(vmas):
mapped = {}
anon = {}
for d in mapped, anon:
# per-perm
for k in perms:
d[k] = {}
d[k]['Size'] = 0
for y in 'Shared', 'Private':
d[k][y] = {}
for z in 'Clean', 'Dirty':
d[k][y][z] = 0
# totals
for y in 'Shared', 'Private':
d[y] = {}
for z in 'Clean', 'Dirty':
d[y][z] = 0
for vma in vmas:
if vma.major == '00' and vma.minor == '00':
d = anon
else:
d = mapped
for y in 'Shared', 'Private':
for z in 'Clean', 'Dirty':
d[vma.perms][y][z] += vma.get(y + '_' + z, 0)
d[y][z] += vma.get(y + '_' + z, 0)
d[vma.perms]['Size'] += vma.get('Size', 0)
return mapped, anon
def values(d, args):
if args:
ret = ()
first = args[0]
for k in first:
ret += values(d[k], args[1:])
return ret
else:
return (d,)
def print_summary(dicts_and_titles):
def desc(title, perms):
ret = {('Anonymous', 'rw-p'): 'Data (malloc, mmap)',
('Anonymous', 'rwxp'): 'Writable code (stack)',
('Mapped', 'r-xp'): 'Code',
('Mapped', 'rwxp'): 'Writable code (jump tables)',
('Mapped', 'r--p'): 'Read-only data',
('Mapped', 'rw-p'): 'Data'}.get((title, perms), None)
if ret:
return ' -- ' + ret
else:
return ''
for d, title in dicts_and_titles:
print title, 'memory:'
print ' Shared Private'
print ' Clean Dirty Clean Dirty'
for k in perms:
if d[k]['Size']:
print (' %s %7d %7d %7d %7d%s'
% ((k,)
+ values(d[k], (('Shared', 'Private'),
('Clean', 'Dirty')))
+ (desc(title, k),)))
print (' total %7d %7d %7d %7d'
% values(d, (('Shared', 'Private'),
('Clean', 'Dirty'))))
print ' ' + '-' * 40
print (' total %7d %7d %7d %7d'
% tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
('Clean', 'Dirty')))
for d, title in dicts_and_titles]))))
def print_stats(pid=None):
if pid is None:
pid = os.getpid()
vmas = parse_smaps(pid)
mapped, anon = make_summary_dicts(vmas)
print_summary(((mapped, "Mapped"), (anon, "Anonymous")))
def linux_memory(since=0.0):
'''Return memory usage in bytes.
'''
return _VmB('VmSize:') - since
vmas = parse_smaps(os.getpid())
mapped, anon = make_summary_dicts(vmas)
dicts_and_titles = ((mapped, "Mapped"), (anon, "Anonymous"))
totals = tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
('Clean', 'Dirty')))
for d, title in dicts_and_titles])))
return (totals[-1]/1024.) - since
def resident(since=0.0):
'''Return resident memory usage in bytes.
'''
return _VmB('VmRSS:') - since
def stacksize(since=0.0):
'''Return stack size in bytes.
'''
return _VmB('VmStk:') - since
## end of http://code.activestate.com/recipes/286222/ }}}
memory = linux_memory
elif iswindows:
import win32process
import win32con
@ -95,7 +191,7 @@ def meminfo(handle):
def win_memory(since=0.0):
info = meminfo(get_handle(os.getpid()))
return info['WorkingSetSize'] - since
return (info['WorkingSetSize']/1024.**2) - since
memory = win_memory