From 53fe026cfe293f8bfa6f30c658e9ebfa979ead6d Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Tue, 9 Feb 2021 14:46:05 -0600 Subject: [PATCH] Basic version of BG % done reporting by stories & chapters. --- calibre-plugin/jobs.py | 41 ++++++++++++++++++----------- fanficfare/adapters/base_adapter.py | 7 +++-- fanficfare/writers/base_writer.py | 24 ++++++++--------- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py index 589feda2..9df5a30a 100644 --- a/calibre-plugin/jobs.py +++ b/calibre-plugin/jobs.py @@ -54,14 +54,19 @@ def do_download_worker(book_list, server = Server(pool_size=cpus) logger.info(options['version']) - total = 0 sites_lists = defaultdict(list) [ sites_lists[x['site']].append(x) for x in book_list if x['good'] ] - total = sum(1 for x in book_list if x['good']) + + totals = {} + # can't do direct assignment in list comprehension? I'm sure it + # makes sense to some pythonista. + # [ totals[x['url']]=0.0 for x in book_list if x['good'] ] + [ totals.update({x['url']:0.0}) for x in book_list if x['good'] ] # logger.debug(sites_lists.keys()) # Queue all the jobs + job_count = 0 for site in sites_lists.keys(): logger.debug(_("Launch background process for site %s")%site) site_list = sites_lists[site] @@ -75,6 +80,7 @@ def do_download_worker(book_list, args=args) job._site_list = site_list server.add_job(job) + job_count += len(site_list) # This server is an arbitrary_n job, so there is a notifier available. # Set the % complete to a small number to avoid the 'unavailable' indicator @@ -88,12 +94,13 @@ def do_download_worker(book_list, # produces a notification. Ignore these. # XXX Can get notifications here? try: - ## assumes one download finished per notification. - ## per chapter tracking remains a distant dream... + ## msg = book['url'] (percent,msg) = job.notifications.get_nowait() - count += 1 - logger.info(_("Finished: %s")%msg) - notification(float(count)/total, _('%(count)d of %(total)d stories finished downloading')%{'count':count,'total':total}) + logger.debug("%s<-%s"%(percent,msg)) + totals[msg] = percent/len(totals) + if percent == 1.0: + count += 1 + notification(max(0.01,sum(totals.values())), _('%(count)d of %(total)d stories finished downloading')%{'count':count,'total':len(totals)}) except Empty: pass job.update() @@ -112,7 +119,7 @@ def do_download_worker(book_list, # Add this job's output to the current log # logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title'])) - if count >= total: + if job_count == count: book_list = sorted(book_list,key=lambda x : x['listorder']) logger.info("\n"+_("Download Results:")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list]))) @@ -158,8 +165,8 @@ def do_download_site(site,book_list,options,merge,notification=lambda x,y:x): retval = [] for book in book_list: logger.info("%s"%book['url']) - retval.append(do_download_for_worker(book,options,merge)) - notification(0.0,book['url'])#float(count)/total, _('%(count)d of %(total)d %(site)s stories finished downloading')%{'count':count,'total':total,'site':site}) + retval.append(do_download_for_worker(book,options,merge,notification)) + notification(1.0,book['url']) return retval def do_download_for_worker(book,options,merge,notification=lambda x,y:x): @@ -207,9 +214,9 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x): adapter.password = book['password'] adapter.setChaptersRange(book['begin'],book['end']) - ## each download starts with a new copy of the cookiejar - ## and basic_cache from the FG process. They are not shared - ## between BG downloads at this time. + ## each site download job starts with a new copy of the + ## cookiejar and basic_cache from the FG process. They + ## are not shared between different sites' BG downloads if configuration.getConfig('use_browser_cache'): if 'browser_cache' in options: configuration.set_browser_cache(options['browser_cache']) @@ -281,7 +288,9 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x): logger.info("write to %s"%outfile) inject_cal_cols(book,story,configuration) - writer.writeStory(outfilename=outfile, forceOverwrite=True) + writer.writeStory(outfilename=outfile, + forceOverwrite=True, + notification=notification) if adapter.story.chapter_error_count > 0: book['comment'] = _('Download %(fileform)s completed, %(failed)s failed chapters, %(total)s total chapters.')%\ @@ -340,7 +349,9 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x): logger.info("write to %s"%outfile) inject_cal_cols(book,story,configuration) - writer.writeStory(outfilename=outfile, forceOverwrite=True) + writer.writeStory(outfilename=outfile, + forceOverwrite=True, + notification=notification) if adapter.story.chapter_error_count > 0: book['comment'] = _('Update %(fileform)s completed, added %(added)s chapters, %(failed)s failed chapters, for %(total)s total.')%\ diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 282d2393..1fa92158 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -191,7 +191,7 @@ class BaseSiteAdapter(Requestable): self.story.setMetadata('numChapters', self.num_chapters()) # Does the download the first time it's called. - def getStory(self): + def getStory(self, notification=lambda x,y:x): if not self.storyDone: self.getStoryMetadataOnly(get_cover=True) @@ -199,6 +199,8 @@ class BaseSiteAdapter(Requestable): if self.oldchaptersmap: self.oldchaptersmap = dict((self.normalize_chapterurl(key), value) for (key, value) in self.oldchaptersmap.items()) + percent = 0.0 + per_step = 1.0/self.story.getChapterCount() for index, chap in enumerate(self.chapterUrls): title = chap['title'] url = chap['url'] @@ -260,7 +262,8 @@ class BaseSiteAdapter(Requestable): passchap['url'] = url passchap['title'] = title passchap['html'] = data - + percent += per_step + notification(percent,self.url) ## XXX -- add chapter text replacement here? self.story.addChapter(passchap, newchap) self.storyDone = True diff --git a/fanficfare/writers/base_writer.py b/fanficfare/writers/base_writer.py index b64351d4..e7b013d0 100644 --- a/fanficfare/writers/base_writer.py +++ b/fanficfare/writers/base_writer.py @@ -157,7 +157,12 @@ class BaseStoryWriter(Requestable): self._write(out,END.substitute(self.story.getAllMetadata())) # if no outstream is given, write to file. - def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False): + def writeStory(self, + outstream=None, + metaonly=False, + outfilename=None, + forceOverwrite=False, + notification=lambda x,y:x): self.metaonly = metaonly if outfilename == None: @@ -194,23 +199,18 @@ class BaseStoryWriter(Requestable): logger.warning("File(%s) Updated(%s) more recently than Story(%s) - Skipping" % (outfilename,fileupdated,lastupdated)) return if not metaonly: - self.story = self.adapter.getStory() # get full story - # now, just - # before writing. - # Fetch before - # opening file. + # get full story now, just before writing. Fetch + # before opening file. + self.story = self.adapter.getStory(notification) outstream = open(outfilename,"wb") else: close=False logger.debug("Save to stream") if not metaonly: - self.story = self.adapter.getStory() # get full story now, - # just before - # writing. Okay if - # double called with - # above, it will only - # fetch once. + # get full story now, just before writing. Okay if double + # called with above, it will only fetch once. + self.story = self.adapter.getStory(notification) if self.getConfig('zip_output'): out = BytesIO() self.zipout = ZipFile(outstream, 'w', compression=ZIP_DEFLATED)