From 1f1128d5a4e8469ea4a947e0fae75d63ad7fea4b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Mar 2011 00:08:02 -0700 Subject: [PATCH] Custom recipes: Store custom recipes in the calibre config directory instead of the library database. This allows scheduling of custom recipes to work with multiple libraries. Note that you may have to re-schedule any existing custom recipes --- src/calibre/gui2/dialogs/scheduler.py | 10 +-- src/calibre/gui2/dialogs/user_profiles.py | 3 +- src/calibre/library/database2.py | 10 --- src/calibre/library/schema_upgrades.py | 18 +++++ src/calibre/web/feeds/recipes/__init__.py | 9 +++ src/calibre/web/feeds/recipes/collection.py | 80 +++++++++++++++++++-- src/calibre/web/feeds/recipes/model.py | 35 ++++----- 7 files changed, 122 insertions(+), 43 deletions(-) diff --git a/src/calibre/gui2/dialogs/scheduler.py b/src/calibre/gui2/dialogs/scheduler.py index 67a13813df..b2561342b8 100644 --- a/src/calibre/gui2/dialogs/scheduler.py +++ b/src/calibre/gui2/dialogs/scheduler.py @@ -411,7 +411,8 @@ def __init__(self, parent, db): QObject.__init__(self, parent) self.internet_connection_failed = False self._parent = parent - self.recipe_model = RecipeModel(db) + self.recipe_model = RecipeModel() + self.db = db self.lock = QMutex(QMutex.Recursive) self.download_queue = set([]) @@ -433,7 +434,9 @@ def __init__(self, parent, db): self.timer.timeout.connect(self.check) self.oldest = gconf['oldest_news'] QTimer.singleShot(5 * 1000, self.oldest_check) - self.database_changed = self.recipe_model.database_changed + + def database_changed(self, db): + self.db = db def oldest_check(self): if self.oldest > 0: @@ -549,7 +552,6 @@ def check(self): if __name__ == '__main__': from calibre.gui2 import is_ok_to_use_qt is_ok_to_use_qt() - from calibre.library.database2 import LibraryDatabase2 - d = SchedulerDialog(RecipeModel(LibraryDatabase2('/home/kovid/documents/library'))) + d = SchedulerDialog(RecipeModel()) d.exec_() diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index fe64deb430..290982caaf 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -366,8 +366,7 @@ def reject(self): if __name__ == '__main__': from calibre.gui2 import is_ok_to_use_qt is_ok_to_use_qt() - from calibre.library.database2 import LibraryDatabase2 from calibre.web.feeds.recipes.model import RecipeModel - d=UserProfiles(None, RecipeModel(LibraryDatabase2('/home/kovid/documents/library'))) + d=UserProfiles(None, RecipeModel()) d.exec_() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 2554df93e6..38b70fc2bf 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1187,12 +1187,6 @@ def doit(ltable, table, ltable_col): self.clean_custom() self.conn.commit() - def get_recipes(self): - return self.conn.get('SELECT id, script FROM feeds') - - def get_recipe(self, id): - return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False) - def get_books_for_category(self, category, id_): ans = set([]) @@ -3113,8 +3107,4 @@ def get_ids_for_custom_book_data(self, name): s = self.conn.get('''SELECT book FROM books_plugin_data WHERE name=?''', (name,)) return [x[0] for x in s] - def get_custom_recipes(self): - for id, title, script in self.conn.get('SELECT id,title,script FROM feeds'): - yield id, title, script - diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index d1f22d379b..3fc9a2368a 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -582,4 +582,22 @@ def upgrade_version_18(self): # statements self.conn.executescript(script) + def upgrade_version_19(self): + recipes = self.conn.get('SELECT id,title,script FROM feeds') + if recipes: + from calibre.web.feeds.recipes import custom_recipes, \ + custom_recipe_filename + bdir = os.path.dirname(custom_recipes.file_path) + for id_, title, script in recipes: + existing = frozenset(map(int, custom_recipes.iterkeys())) + if id_ in existing: + id_ = max(existing) + 1000 + id_ = str(id_) + fname = custom_recipe_filename(id_, title) + custom_recipes[id_] = (title, fname) + if isinstance(script, unicode): + script = script.encode('utf-8') + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) + diff --git a/src/calibre/web/feeds/recipes/__init__.py b/src/calibre/web/feeds/recipes/__init__.py index a72f500862..bfb46fa799 100644 --- a/src/calibre/web/feeds/recipes/__init__.py +++ b/src/calibre/web/feeds/recipes/__init__.py @@ -10,6 +10,7 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ptempfile import PersistentTemporaryDirectory from calibre import __appname__, english_sort +from calibre.utils.config import JSONConfig BeautifulSoup, time, english_sort @@ -17,6 +18,14 @@ CalibrePeriodical) _tdir = None _crep = 0 + +custom_recipes = JSONConfig('custom_recipes/index.json') + +def custom_recipe_filename(id_, title): + from calibre.utils.filenames import ascii_filename + return ascii_filename(title[:50]) + \ + ('_%s.recipe'%id_) + def compile_recipe(src): ''' Compile the code in src and return the first object that is a recipe or profile. diff --git a/src/calibre/web/feeds/recipes/collection.py b/src/calibre/web/feeds/recipes/collection.py index aa0051ca37..7082f780e6 100644 --- a/src/calibre/web/feeds/recipes/collection.py +++ b/src/calibre/web/feeds/recipes/collection.py @@ -88,19 +88,89 @@ def serialize_builtin_recipes(): def get_builtin_recipe_collection(): return etree.parse(P('builtin_recipes.xml')).getroot() -def get_custom_recipe_collection(db): - from calibre.web.feeds.recipes import compile_recipe +def get_custom_recipe_collection(*args): + from calibre.web.feeds.recipes import compile_recipe, \ + custom_recipes + bdir = os.path.dirname(custom_recipes.file_path) rmap = {} - for id, title, recipe in db.get_custom_recipes(): + for id_, x in custom_recipes.iteritems(): + title, fname = x + recipe = os.path.join(bdir, fname) try: + recipe = open(recipe, 'rb').read().decode('utf-8') recipe_class = compile_recipe(recipe) if recipe_class is not None: - rmap['custom:%d'%id] = recipe_class + rmap['custom:%s'%id_] = recipe_class except: + import traceback + traceback.print_exc() continue - return etree.fromstring(serialize_collection(rmap)) + +def update_custom_recipe(id_, title, script): + from calibre.web.feeds.recipes import custom_recipes, \ + custom_recipe_filename + id_ = str(int(id_)) + existing = custom_recipes.get(id_, None) + bdir = os.path.dirname(custom_recipes.file_path) + + if existing is None: + fname = custom_recipe_filename(id_, title) + else: + fname = existing[1] + if isinstance(script, unicode): + script = script.encode('utf-8') + + custom_recipes[id_] = (title, fname) + + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) + + +def add_custom_recipe(title, script): + from calibre.web.feeds.recipes import custom_recipes, \ + custom_recipe_filename + id_ = 1000 + keys = tuple(map(int, custom_recipes.iterkeys())) + if keys: + id_ = max(keys)+1 + id_ = str(id_) + bdir = os.path.dirname(custom_recipes.file_path) + + fname = custom_recipe_filename(id_, title) + if isinstance(script, unicode): + script = script.encode('utf-8') + + custom_recipes[id_] = (title, fname) + + with open(os.path.join(bdir, fname), 'wb') as f: + f.write(script) + + +def remove_custom_recipe(id_): + from calibre.web.feeds.recipes import custom_recipes + id_ = str(int(id_)) + existing = custom_recipes.get(id_, None) + if existing is not None: + bdir = os.path.dirname(custom_recipes.file_path) + fname = existing[1] + del custom_recipes[id_] + try: + os.remove(os.path.join(bdir, fname)) + except: + pass + +def get_custom_recipe(id_): + from calibre.web.feeds.recipes import custom_recipes + id_ = str(int(id_)) + existing = custom_recipes.get(id_, None) + if existing is not None: + bdir = os.path.dirname(custom_recipes.file_path) + fname = existing[1] + with open(os.path.join(bdir, fname), 'rb') as f: + return f.read().decode('utf-8') + def get_builtin_recipe_titles(): return [r.get('title') for r in get_builtin_recipe_collection()] diff --git a/src/calibre/web/feeds/recipes/model.py b/src/calibre/web/feeds/recipes/model.py index 553fdcc3c3..19e73dd5f8 100644 --- a/src/calibre/web/feeds/recipes/model.py +++ b/src/calibre/web/feeds/recipes/model.py @@ -9,14 +9,15 @@ import os, copy from PyQt4.Qt import QAbstractItemModel, QVariant, Qt, QColor, QFont, QIcon, \ - QModelIndex, QMetaObject, pyqtSlot, pyqtSignal + QModelIndex, pyqtSignal from calibre.utils.search_query_parser import SearchQueryParser from calibre.gui2 import NONE from calibre.utils.localization import get_language from calibre.web.feeds.recipes.collection import \ get_builtin_recipe_collection, get_custom_recipe_collection, \ - SchedulerConfig, download_builtin_recipe + SchedulerConfig, download_builtin_recipe, update_custom_recipe, \ + add_custom_recipe, remove_custom_recipe, get_custom_recipe from calibre.utils.pyparsing import ParseException class NewsTreeItem(object): @@ -122,26 +123,15 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser): LOCATIONS = ['all'] searched = pyqtSignal(object) - def __init__(self, db, *args): + def __init__(self, *args): QAbstractItemModel.__init__(self, *args) SearchQueryParser.__init__(self, locations=['all']) - self.db = db self.default_icon = QVariant(QIcon(I('news.png'))) self.custom_icon = QVariant(QIcon(I('user_profile.png'))) self.builtin_recipe_collection = get_builtin_recipe_collection() self.scheduler_config = SchedulerConfig() self.do_refresh() - @pyqtSlot() - def do_database_change(self): - self.db = self.newdb - self.newdb = None - self.do_refresh() - - def database_changed(self, db): - self.newdb = db - QMetaObject.invokeMethod(self, 'do_database_change', Qt.QueuedConnection) - def get_builtin_recipe(self, urn, download=True): if download: try: @@ -158,23 +148,24 @@ def get_recipe(self, urn, download=True): if recipe.get('id', False) == urn: if coll is self.builtin_recipe_collection: return self.get_builtin_recipe(urn[8:], download=download) - return self.db.get_feed(int(urn[len('custom:'):])) + return get_custom_recipe(int(urn[len('custom:'):])) def update_custom_recipe(self, urn, title, script): - self.db.update_feed(int(urn[len('custom:'):]), script, title) - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + id_ = int(urn[len('custom:'):]) + update_custom_recipe(id_, title, script) + self.custom_recipe_collection = get_custom_recipe_collection() def add_custom_recipe(self, title, script): - self.db.add_feed(title, script) - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + add_custom_recipe(title, script) + self.custom_recipe_collection = get_custom_recipe_collection() def remove_custom_recipes(self, urns): ids = [int(x[len('custom:'):]) for x in urns] - self.db.remove_feeds(ids) - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + for id_ in ids: remove_custom_recipe(id_) + self.custom_recipe_collection = get_custom_recipe_collection() def do_refresh(self, restrict_to_urns=set([])): - self.custom_recipe_collection = get_custom_recipe_collection(self.db) + self.custom_recipe_collection = get_custom_recipe_collection() def factory(cls, parent, *args): args = list(args)