Backup_Repos/FanFicFare
1
0
Fork 0
mirror of https://github.com/JimmXinu/FanFicFare.git synced 2026-05-08 12:36:11 +02:00
Code Issues Projects Releases Wiki Activity

Somewhat better consolidated perf profiling

Browse source
This commit is contained in:
Jim Miller 2026-04-10 21:44:09 -05:00
parent 08bae8d9be
commit d46dc76ae1
3 changed files with 64 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
calibre-plugin/jobs.py
View file

@ -24,34 +24,12 @@ try:
except NameError:
pass # load_translations() added in calibre 1.9
from io import StringIO
import cProfile, pstats
from pstats import SortKey
def do_cprofile(func):
def profiled_func(*args, **kwargs):
profile = cProfile.Profile()
try:
profile.enable()
result = func(*args, **kwargs)
profile.disable()
return result
finally:
# profile.print_stats()
s = StringIO()
sortby = SortKey.CUMULATIVE
ps = pstats.Stats(profile, stream=s).sort_stats(sortby)
ps.print_stats(20)
print(s.getvalue())
return profiled_func
# ------------------------------------------------------------------------------
#
# Functions to perform downloads using worker jobs
#
# ------------------------------------------------------------------------------
@do_cprofile
def do_download_worker_single(site,
book_list,
options,
@ -66,34 +44,44 @@ def do_download_worker_single(site,
print_basic_debug_info(sys.stderr)
notification(0.01, _('Downloading FanFiction Stories'))
from calibre_plugins.fanficfare_plugin import FanFicFareBase
fffbase = FanFicFareBase(options['plugin_path'])
with fffbase: # so the sys.path was modified while loading the
# plug impl.
from fanficfare.fff_profile import do_cprofile
count = 0
totals = {}
# can't do direct assignment in list comprehension? I'm sure it
# makes sense to some pythonista.
# [ totals[x['url']]=0.0 for x in book_list if x['good'] ]
[ totals.update({x['url']:0.0}) for x in book_list if x['good'] ]
# logger.debug(sites_lists.keys())
## extra function just so I can easily use the same
## @do_cprofile decorator
@do_cprofile
def profiled_func():
count = 0
totals = {}
# can't do direct assignment in list comprehension? I'm sure it
# makes sense to some pythonista.
# [ totals[x['url']]=0.0 for x in book_list if x['good'] ]
[ totals.update({x['url']:0.0}) for x in book_list if x['good'] ]
# logger.debug(sites_lists.keys())
def do_indiv_notif(percent,msg):
totals[msg] = percent/len(totals)
notification(max(0.01,sum(totals.values())), _('%(count)d of %(total)d stories finished downloading')%{'count':count,'total':len(totals)})
def do_indiv_notif(percent,msg):
totals[msg] = percent/len(totals)
notification(max(0.01,sum(totals.values())), _('%(count)d of %(total)d stories finished downloading')%{'count':count,'total':len(totals)})
do_list = []
done_list = []
logger.info("\n\n"+_("Downloading FanFiction Stories")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
## pass failures from metadata through bg job so all results are
## together.
for book in book_list:
if book['good']:
do_list.append(book)
else:
done_list.append(book)
for book in do_list:
# logger.info("%s"%book['url'])
done_list.append(do_download_for_worker(book,options,merge,do_indiv_notif))
count += 1
return finish_download(done_list)
do_list = []
done_list = []
logger.info("\n\n"+_("Downloading FanFiction Stories")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
## pass failures from metadata through bg job so all results are
## together.
for book in book_list:
if book['good']:
do_list.append(book)
else:
done_list.append(book)
for book in do_list:
# logger.info("%s"%book['url'])
done_list.append(do_download_for_worker(book,options,merge,do_indiv_notif))
count += 1
return finish_download(done_list)
return profiled_func()
def finish_download(donelist):
book_list = sorted(donelist,key=lambda x : x['listorder'])

20
fanficfare/epubutils.py
View file

@ -22,25 +22,7 @@ from io import BytesIO
FONT_EXTS = ('ttf','otf','woff','woff2')
from io import StringIO
import cProfile, pstats
from pstats import SortKey
def do_cprofile(func):
def profiled_func(*args, **kwargs):
profile = cProfile.Profile()
try:
profile.enable()
result = func(*args, **kwargs)
profile.disable()
return result
finally:
# profile.sort_stats(SortKey.CUMULATIVE).print_stats(20)
s = StringIO()
sortby = SortKey.CUMULATIVE
ps = pstats.Stats(profile, stream=s).sort_stats(sortby)
ps.print_stats(20)
print(s.getvalue())
return profiled_func
from fanficfare.fff_profile import do_cprofile
import bs4

48
fanficfare/fff_profile.py
View file

@ -13,23 +13,31 @@
# limitations under the License.
#
from io import StringIO
import cProfile, pstats
from pstats import SortKey
def do_cprofile(func):
def profiled_func(*args, **kwargs):
profile = cProfile.Profile()
try:
profile.enable()
result = func(*args, **kwargs)
profile.disable()
return result
finally:
# profile.print_stats()
s = StringIO()
sortby = SortKey.CUMULATIVE
ps = pstats.Stats(profile, stream=s).sort_stats(sortby)
ps.print_stats(20)
print(s.getvalue())
return profiled_func
## not compatibly with py2 SortKey not available.
import sys
if sys.version_info >= (2, 9):
from io import StringIO
import cProfile, pstats
from pstats import SortKey
def do_cprofile(func):
def profiled_func(*args, **kwargs):
profile = cProfile.Profile()
try:
profile.enable()
result = func(*args, **kwargs)
profile.disable()
return result
finally:
# profile.print_stats()
s = StringIO()
sortby = SortKey.CUMULATIVE
ps = pstats.Stats(profile, stream=s).sort_stats(sortby)
ps.print_stats(20)
print(s.getvalue())
return profiled_func
else:
## no-nothing for py2
def do_cprofile(func):
def profiled_func(*args, **kwargs):
return func(*args, **kwargs)
return profiled_func