Compare commits

...

13 commits

11 changed files with 257 additions and 38 deletions

View file

@ -33,7 +33,7 @@ except NameError:
from calibre.customize import InterfaceActionBase
# pulled out from FanFicFareBase for saving in prefs.py
__version__ = (4, 57, 0)
__version__ = (4, 57, 7)
## Apparently the name for this class doesn't matter--it was still
## 'demo' for the first few versions.

View file

@ -1599,6 +1599,8 @@ chaptertitles:Prologue,Chapter 1\, Xenos on Cinnabar,Chapter 2\, Sinmay on Kinti
[adult-fanfiction.org]
use_basic_cache:true
extra_valid_entries:eroticatags,disclaimer
eroticatags_label:Erotica Tags
disclaimer_label:Disclaimer
@ -1717,13 +1719,13 @@ make_linkhtml_entries:series00,series01,series02,series03,collections
## hardcoded to include the site specific metadata freeformtags &
## ao3categories in the standard metadata field genre. By making it
## configurable, users can change it.
include_in_genre: freeformtags, ao3categories
include_in_genre: genre, freeformtags, ao3categories
## AO3 uses the word 'category' differently than most sites. The
## adapter used to be hardcoded to include the site specific metadata
## fandom in the standard metadata field category. By making it
## configurable, users can change it.
include_in_category:fandoms
include_in_category:category,fandoms
## freeformtags was previously typo'ed as freefromtags. This way,
## freefromtags will still work for people who've used it.
@ -1932,7 +1934,7 @@ make_linkhtml_entries:translators,betas
## For most sites, 'category' is the fandom, but fanfics.me has
## fandoms and a separate category. By making it configurable, users
## can change it.
include_in_category:fandoms
include_in_category:category,fandoms
[fanfictalk.com]
use_basic_cache:true
@ -2708,13 +2710,13 @@ make_linkhtml_entries:series00,series01,series02,series03,collections
## hardcoded to include the site specific metadata freeformtags &
## ao3categories in the standard metadata field genre. By making it
## configurable, users can change it.
include_in_genre: freeformtags, ao3categories
include_in_genre: genre, freeformtags, ao3categories
## OTW uses the word 'category' differently than most sites. The
## adapter used to be hardcoded to include the site specific metadata
## fandom in the standard metadata field category. By making it
## configurable, users can change it.
include_in_category:fandoms
include_in_category:category,fandoms
## freeformtags was previously typo'ed as freefromtags. This way,
## freefromtags will still work for people who've used it.
@ -3015,13 +3017,13 @@ make_linkhtml_entries:series00,series01,series02,series03,collections
## hardcoded to include the site specific metadata freeformtags &
## ao3categories in the standard metadata field genre. By making it
## configurable, users can change it.
include_in_genre: freeformtags, ao3categories
include_in_genre: genre, freeformtags, ao3categories
## OTW uses the word 'category' differently than most sites. The
## adapter used to be hardcoded to include the site specific metadata
## fandom in the standard metadata field category. By making it
## configurable, users can change it.
include_in_category:fandoms
include_in_category:category,fandoms
## freeformtags was previously typo'ed as freefromtags. This way,
## freefromtags will still work for people who've used it.
@ -3150,8 +3152,8 @@ bookmarkmemo_label:ブックマークメモ
bookmarkprivate_label:非公開ブックマーク
subscribed_label:更新通知
include_in_genre: fullgenre
#include_in_genre: biggenre, smallgenre
include_in_genre: genre, fullgenre
#include_in_genre: genre, biggenre, smallgenre
## adds to titlepage_entries instead of replacing it.
#extra_titlepage_entries: fullgenre,biggenre,smallgenre,imprint,freeformtags,comments,reviews,bookmarks,ratingpoints,overallpoints,bookmarked,bookmarkcategory,bookmarkmemo,bookmarkprivate,subscribed
@ -3394,13 +3396,13 @@ make_linkhtml_entries:series00,series01,series02,series03,collections
## hardcoded to include the site specific metadata freeformtags &
## ao3categories in the standard metadata field genre. By making it
## configurable, users can change it.
include_in_genre: freeformtags, ao3categories
include_in_genre: genre, freeformtags, ao3categories
## OTW uses the word 'category' differently than most sites. The
## adapter used to be hardcoded to include the site specific metadata
## fandom in the standard metadata field category. By making it
## configurable, users can change it.
include_in_category:fandoms
include_in_category:category,fandoms
## freeformtags was previously typo'ed as freefromtags. This way,
## freefromtags will still work for people who've used it.
@ -3531,7 +3533,7 @@ upvotes_label:Upvotes
subscribers_label:Subscribers
views_label:Views
include_in_category:tags
include_in_category:category,tags
#extra_titlepage_entries:upvotes,subscribers,views
@ -3667,13 +3669,13 @@ make_linkhtml_entries:series00,series01,series02,series03,collections
## hardcoded to include the site specific metadata freeformtags &
## ao3categories in the standard metadata field genre. By making it
## configurable, users can change it.
include_in_genre: freeformtags, ao3categories
include_in_genre: genre, freeformtags, ao3categories
## OTW uses the word 'category' differently than most sites. The
## adapter used to be hardcoded to include the site specific metadata
## fandom in the standard metadata field category. By making it
## configurable, users can change it.
include_in_category:fandoms
include_in_category:category,fandoms
## freeformtags was previously typo'ed as freefromtags. This way,
## freefromtags will still work for people who've used it.

View file

@ -53,6 +53,9 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
#Setting the 'Zone' for each "Site"
self.zone = self.parsedUrl.netloc.replace('.fanficauthors.net','')
# site change .nsns to -nsns
self.zone = self.zone.replace('.nsns','-nsns')
# normalized story URL.
self._setURL('https://{0}.{1}/{2}/'.format(
self.zone, self.getBaseDomain(), self.story.getMetadata('storyId')))
@ -79,7 +82,10 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
@classmethod
def getAcceptDomains(cls):
# need both .nsns(old) and -nsns(new) because it's a domain
# change, not just URL change.
return ['aaran-st-vines.nsns.fanficauthors.net',
'aaran-st-vines-nsns.fanficauthors.net',
'abraxan.fanficauthors.net',
'bobmin.fanficauthors.net',
'canoncansodoff.fanficauthors.net',
@ -95,9 +101,12 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
'jeconais.fanficauthors.net',
'kinsfire.fanficauthors.net',
'kokopelli.nsns.fanficauthors.net',
'kokopelli-nsns.fanficauthors.net',
'ladya.nsns.fanficauthors.net',
'ladya-nsns.fanficauthors.net',
'lorddwar.fanficauthors.net',
'mrintel.nsns.fanficauthors.net',
'mrintel-nsns.fanficauthors.net',
'musings-of-apathy.fanficauthors.net',
'ruskbyte.fanficauthors.net',
'seelvor.fanficauthors.net',
@ -108,7 +117,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
################################################################################################
@classmethod
def getSiteExampleURLs(self):
return ("https://aaran-st-vines.nsns.fanficauthors.net/A_Story_Name/ "
return ("https://aaran-st-vines-nsns.fanficauthors.net/A_Story_Name/ "
+ "https://abraxan.fanficauthors.net/A_Story_Name/ "
+ "https://bobmin.fanficauthors.net/A_Story_Name/ "
+ "https://canoncansodoff.fanficauthors.net/A_Story_Name/ "
@ -123,10 +132,10 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
+ "https://jbern.fanficauthors.net/A_Story_Name/ "
+ "https://jeconais.fanficauthors.net/A_Story_Name/ "
+ "https://kinsfire.fanficauthors.net/A_Story_Name/ "
+ "https://kokopelli.nsns.fanficauthors.net/A_Story_Name/ "
+ "https://ladya.nsns.fanficauthors.net/A_Story_Name/ "
+ "https://kokopelli-nsns.fanficauthors.net/A_Story_Name/ "
+ "https://ladya-nsns.fanficauthors.net/A_Story_Name/ "
+ "https://lorddwar.fanficauthors.net/A_Story_Name/ "
+ "https://mrintel.nsns.fanficauthors.net/A_Story_Name/ "
+ "https://mrintel-nsns.fanficauthors.net/A_Story_Name/ "
+ "https://musings-of-apathy.fanficauthors.net/A_Story_Name/ "
+ "https://ruskbyte.fanficauthors.net/A_Story_Name/ "
+ "https://seelvor.fanficauthors.net/A_Story_Name/ "
@ -136,8 +145,16 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
################################################################################################
def getSiteURLPattern(self):
## .nsns kept here to match both . and -
return r'https?://(aaran-st-vines.nsns|abraxan|bobmin|canoncansodoff|chemprof|copperbadge|crys|deluded-musings|draco664|fp|frenchsession|ishtar|jbern|jeconais|kinsfire|kokopelli.nsns|ladya.nsns|lorddwar|mrintel.nsns|musings-of-apathy|ruskbyte|seelvor|tenhawk|viridian|whydoyouneedtoknow)\.fanficauthors\.net/([a-zA-Z0-9_]+)/'
@classmethod
def get_section_url(cls,url):
## only changing .nsns to -nsns and only when part of the
## domain.
url = url.replace('.nsns.fanficauthors.net','-nsns.fanficauthors.net')
return url
################################################################################################
def doExtractChapterUrlsAndMetadata(self, get_cover=True):

View file

@ -22,6 +22,7 @@ from .base_browsercache import BaseBrowserCache, CACHE_DIR_CONFIG
from .browsercache_simple import SimpleCache
from .browsercache_blockfile import BlockfileCache
from .browsercache_firefox2 import FirefoxCache2
from .browsercache_sqldb import SqldbCache
import logging
logger = logging.getLogger(__name__)
@ -34,12 +35,13 @@ class BrowserCache(object):
def __init__(self, site, getConfig_fn, getConfigList_fn):
"""Constructor for BrowserCache"""
# import of child classes have to be inside the def to avoid circular import error
for browser_cache_class in [SimpleCache, BlockfileCache, FirefoxCache2]:
for browser_cache_class in [SimpleCache, BlockfileCache, FirefoxCache2, SqldbCache]:
self.browser_cache_impl = browser_cache_class.new_browser_cache(site,
getConfig_fn,
getConfigList_fn)
if self.browser_cache_impl is not None:
break
logger.debug("Not using Browser Cache Class %s"%browser_cache_class)
if self.browser_cache_impl is None:
raise BrowserCacheException("%s is not set, or directory does not contain a known browser cache type: '%s'"%
(CACHE_DIR_CONFIG,getConfig_fn(CACHE_DIR_CONFIG)))

View file

@ -90,18 +90,23 @@ class BlockfileCache(BaseChromiumCache):
def is_cache_dir(cache_dir):
"""Return True only if a directory is a valid Cache for this class"""
if not os.path.isdir(cache_dir):
logger.debug("Cache dir not found")
return False
index_path = os.path.join(cache_dir, "index")
if not os.path.isfile(index_path):
logger.debug("index file not found")
return False
with share_open(index_path, 'rb') as index_file:
if struct.unpack('I', index_file.read(4))[0] != INDEX_MAGIC_NUMBER:
logger.debug("index file failed magic number check")
return False
data0_path = os.path.join(cache_dir, "data_0")
if not os.path.isfile(data0_path):
logger.debug("data_0 file not found")
return False
with share_open(data0_path, 'rb') as data0_file:
if struct.unpack('I', data0_file.read(4))[0] != BLOCK_MAGIC_NUMBER:
logger.debug("data_0 failed magic number check")
return False
return True

View file

@ -68,6 +68,7 @@ class FirefoxCache2(BaseBrowserCache):
"""Return True only if a directory is a valid Cache for this class"""
# logger.debug("\n\n1Starting cache check\n\n")
if not os.path.isdir(cache_dir):
logger.debug("Cache dir not found")
return False
## check at least one entry file exists.
for en_fl in glob.iglob(os.path.join(cache_dir, 'entries', '????????????????????????????????????????')):
@ -75,6 +76,7 @@ class FirefoxCache2(BaseBrowserCache):
k = _validate_entry_file(en_fl)
if k is not None:
return True
logger.debug("No valid cache files found")
return False
def make_keys(self,url):

View file

@ -76,15 +76,19 @@ class SimpleCache(BaseChromiumCache):
def is_cache_dir(cache_dir):
"""Return True only if a directory is a valid Cache for this class"""
if not os.path.isdir(cache_dir):
logger.debug("Cache dir not found")
return False
index_file = os.path.join(cache_dir, "index")
if not (os.path.isfile(index_file) and os.path.getsize(index_file) == 24):
if not os.path.isfile(index_file) or os.path.getsize(index_file) > 24:
logger.debug("index file not found or too big(%s)"%os.path.getsize(index_file))
return False
real_index_file = os.path.join(cache_dir, "index-dir", "the-real-index")
if not os.path.isfile(real_index_file):
logger.debug("real_index_file not found")
return False
with share_open(real_index_file, 'rb') as index_file:
if struct.unpack('QQ', index_file.read(16))[1] != THE_REAL_INDEX_MAGIC_NUMBER:
logger.debug("real_index_file failed magic number check")
return False
try:
# logger.debug("\n\nStarting cache check\n\n")
@ -92,9 +96,11 @@ class SimpleCache(BaseChromiumCache):
k = _validate_entry_file(en_fl)
if k is not None:
return True
except SimpleCacheException:
except SimpleCacheException as sce:
# raise
logger.debug(sce)
return False
logger.debug("No valid cache files found")
return False
def get_data_key_impl(self, url, key):

View file

@ -0,0 +1,185 @@
# -*- coding: utf-8 -*-
# Copyright 2026 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import absolute_import
import os
import apsw
import ctypes
# note share_open (on windows CLI) is implicitly readonly.
from .share_open import share_open
from .base_chromium import BaseChromiumCache
from .chromagnon import SuperFastHash
import logging
logger = logging.getLogger(__name__)
class SqldbCache(BaseChromiumCache):
"""Class to access data stream in Chrome Disk Sqldb Cache format cache files"""
def __init__(self, *args, **kargs):
"""Constructor for SqldbCache"""
super(SqldbCache,self).__init__(*args, **kargs)
logger.debug("Using SqldbCache")
# def scan_cache_keys(self):
## XXX will impl a scan if and when needed. It's a lot easier
## to peek inside an sqlite
@staticmethod
def is_cache_dir(cache_dir):
"""Return True only if a directory is a valid Cache for this class"""
if not os.path.isdir(cache_dir):
logger.debug("Cache dir not found")
return False
index_path = os.path.join(cache_dir, "index")
if not os.path.isfile(index_path):
logger.debug("index file not found")
return False
sqldb0_path = os.path.join(cache_dir, "sqldb0")
if not os.path.isfile(sqldb0_path):
logger.debug("sqldb0 file not found")
return False
## XXX check schema of db?
return True
## XXX others uses share_open() - will sqlite open work concurrently?
def get_data_key_impl(self, url, key):
"""
returns location, entry age(unix epoch), content-encoding and
raw(compressed) data
"""
location, age, encoding, data = '', None, None, None
qstr = 'SELECT last_used, head, blob FROM resources as r join blobs as b on b.res_id=r.res_id where cache_key_hash=?'
cache_key_hash = _key_hash(key)
logger.debug(" key:%s"%key)
logger.debug("cache_key_hash:%s"%cache_key_hash)
## XXX worth optimizing to keep sql conn open?
from ..six.moves.urllib.request import pathname2url
fileuri = os.path.join(self.cache_dir, "sqldb0")# pathname2url()
logger.debug(fileuri)
shareopenVFS = ShareOpenVFS()
logger.debug("VFS available %s"% apsw.vfs_names())
with apsw.Connection("file:"+fileuri+"?immutable=1",
flags=apsw.SQLITE_OPEN_READONLY | apsw.SQLITE_OPEN_URI,
vfs=shareopenVFS.vfs_name
) as db:
logger.debug("db flags:%xd"%db.open_flags)
logger.debug("db vfs:%s"%db.open_vfs)
for last, head, blob in db.execute(qstr,[cache_key_hash]):
row_age = self.make_age(last)
if age and row_age < age:
logger.debug("skipping an older row for same hash")
break
age = row_age
logger.debug("age from last_used:%s"%age)
## cheesy way to pull out the http headers, inspired
## by equal cheese in chromagnon/cacheData.py. Only
## actually care about location &content-encoding,
## ignore the rest.
head = head[head.index(b'HTTP'):]
head = head[:head.index(b'\x00\x00')]
# logger.debug(head)
for line in head.split(b'\0'):
logger.debug(line)
if b'content-encoding' in line.lower():
encoding = line.split(b':')[1].strip().lower()
logger.debug("encoding from header:%s"%encoding)
if b'location' in line.lower():
location = b':'.join(line.split(b':')[1:]).strip()
logger.debug("location from header:%s"%encoding)
## XXX might need entry age from header, too.
## Hoping db last_used is equiv.
data = blob
if data:
return (location, age, encoding, data)
else:
return None
## calculate SuperFashHash, but the sql saved it signed.
def _key_hash(key):
unsigned_hash = SuperFastHash.superFastHash(key)
number = unsigned_hash & 0xFFFFFFFF
return ctypes.c_int32(number).value
class ShareOpenVFS(apsw.VFS):
def __init__(self):
self.vfs_name = 'shareopen'
super().__init__(name=self.vfs_name, base='')
def xAccess(self, pathname, flags):
return True
def xFullPathname(self, filename):
return filename
def xDelete(self, filename, syncdir):
logger.debug("xDelete NOT DELETING")
pass
def xOpen(self, name, flags):
return ShareOpenVFSFile(name, flags)
class ShareOpenVFSFile:
def __init__(self, name, flags):
self.filename = name.filename() if isinstance(name, apsw.URIFilename) else name
self.filename = os.path.normpath(self.filename)
logger.debug("Doing share open(%s)"%self.filename)
self.file = share_open(self.filename, 'rb')
def xRead(self, amount, offset):
self.file.seek(offset, 0)
return self.file.read(amount)
def xFileSize(self):
return os.stat(self.filename).st_size
def xClose(self):
self.file.close()
def xSectorSize(self):
return 0
def xFileControl(self, *args):
return False
def xCheckReservedLock(self):
return False
def xLock(self, level):
pass
def xUnlock(self, level):
pass
def xSync(self, flags):
return True
def xTruncate(self, newsize):
logger.debug("xTruncate NOT TRUNCING")
pass
def xWrite(self, data, offset):
logger.debug("xWrite NOT WRITING")
pass

View file

@ -27,7 +27,7 @@ import pprint
import string
import os, sys, platform
version="4.57.0"
version="4.57.7"
os.environ['CURRENT_VERSION_ID']=version
global_cache = 'global_cache'

View file

@ -1712,13 +1712,13 @@ make_linkhtml_entries:series00,series01,series02,series03,collections
## hardcoded to include the site specific metadata freeformtags &
## ao3categories in the standard metadata field genre. By making it
## configurable, users can change it.
include_in_genre: freeformtags, ao3categories
include_in_genre: genre, freeformtags, ao3categories
## AO3 uses the word 'category' differently than most sites. The
## adapter used to be hardcoded to include the site specific metadata
## fandom in the standard metadata field category. By making it
## configurable, users can change it.
include_in_category:fandoms
include_in_category:category,fandoms
## freeformtags was previously typo'ed as freefromtags. This way,
## freefromtags will still work for people who've used it.
@ -1927,7 +1927,7 @@ make_linkhtml_entries:translators,betas
## For most sites, 'category' is the fandom, but fanfics.me has
## fandoms and a separate category. By making it configurable, users
## can change it.
include_in_category:fandoms
include_in_category:category,fandoms
[fanfictalk.com]
use_basic_cache:true
@ -2703,13 +2703,13 @@ make_linkhtml_entries:series00,series01,series02,series03,collections
## hardcoded to include the site specific metadata freeformtags &
## ao3categories in the standard metadata field genre. By making it
## configurable, users can change it.
include_in_genre: freeformtags, ao3categories
include_in_genre: genre, freeformtags, ao3categories
## OTW uses the word 'category' differently than most sites. The
## adapter used to be hardcoded to include the site specific metadata
## fandom in the standard metadata field category. By making it
## configurable, users can change it.
include_in_category:fandoms
include_in_category:category,fandoms
## freeformtags was previously typo'ed as freefromtags. This way,
## freefromtags will still work for people who've used it.
@ -3010,13 +3010,13 @@ make_linkhtml_entries:series00,series01,series02,series03,collections
## hardcoded to include the site specific metadata freeformtags &
## ao3categories in the standard metadata field genre. By making it
## configurable, users can change it.
include_in_genre: freeformtags, ao3categories
include_in_genre: genre, freeformtags, ao3categories
## OTW uses the word 'category' differently than most sites. The
## adapter used to be hardcoded to include the site specific metadata
## fandom in the standard metadata field category. By making it
## configurable, users can change it.
include_in_category:fandoms
include_in_category:category,fandoms
## freeformtags was previously typo'ed as freefromtags. This way,
## freefromtags will still work for people who've used it.
@ -3145,8 +3145,8 @@ bookmarkmemo_label:ブックマークメモ
bookmarkprivate_label:非公開ブックマーク
subscribed_label:更新通知
include_in_genre: fullgenre
#include_in_genre: biggenre, smallgenre
include_in_genre: genre, fullgenre
#include_in_genre: genre, biggenre, smallgenre
## adds to titlepage_entries instead of replacing it.
#extra_titlepage_entries: fullgenre,biggenre,smallgenre,imprint,freeformtags,comments,reviews,bookmarks,ratingpoints,overallpoints,bookmarked,bookmarkcategory,bookmarkmemo,bookmarkprivate,subscribed
@ -3389,13 +3389,13 @@ make_linkhtml_entries:series00,series01,series02,series03,collections
## hardcoded to include the site specific metadata freeformtags &
## ao3categories in the standard metadata field genre. By making it
## configurable, users can change it.
include_in_genre: freeformtags, ao3categories
include_in_genre: genre, freeformtags, ao3categories
## OTW uses the word 'category' differently than most sites. The
## adapter used to be hardcoded to include the site specific metadata
## fandom in the standard metadata field category. By making it
## configurable, users can change it.
include_in_category:fandoms
include_in_category:category,fandoms
## freeformtags was previously typo'ed as freefromtags. This way,
## freefromtags will still work for people who've used it.
@ -3526,7 +3526,7 @@ upvotes_label:Upvotes
subscribers_label:Subscribers
views_label:Views
include_in_category:tags
include_in_category:category,tags
#extra_titlepage_entries:upvotes,subscribers,views
@ -3662,13 +3662,13 @@ make_linkhtml_entries:series00,series01,series02,series03,collections
## hardcoded to include the site specific metadata freeformtags &
## ao3categories in the standard metadata field genre. By making it
## configurable, users can change it.
include_in_genre: freeformtags, ao3categories
include_in_genre: genre, freeformtags, ao3categories
## OTW uses the word 'category' differently than most sites. The
## adapter used to be hardcoded to include the site specific metadata
## fandom in the standard metadata field category. By making it
## configurable, users can change it.
include_in_category:fandoms
include_in_category:category,fandoms
## freeformtags was previously typo'ed as freefromtags. This way,
## freefromtags will still work for people who've used it.

View file

@ -16,7 +16,7 @@ name = "FanFicFare" # Required
#
# For a discussion on single-sourcing the version, see
# https://packaging.python.org/guides/single-sourcing-package-version/
version = "4.57.0"
version = "4.57.7"
# This is a one-line description or tagline of what your project does. This
# corresponds to the "Summary" metadata field: