From c0911f16262adf4b4dbc40dca92bf2e772cf32a8 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Thu, 7 Nov 2019 15:35:04 +1100 Subject: [PATCH] Stop tasks and show task progress (#181) * Add job status to tasks page * Add support for stopping task * Show progress of some tasks --- .../queries/settings/metadata.graphql | 12 ++ graphql/documents/subscriptions.graphql | 6 +- graphql/schema/schema.graphql | 5 +- graphql/schema/types/metadata.graphql | 6 + pkg/api/resolver_query_metadata.go | 15 +++ pkg/api/resolver_subscription_metadata.go | 20 ++- pkg/manager/job_status.go | 19 +++ pkg/manager/manager.go | 4 +- pkg/manager/manager_subscription_handler.go | 36 ------ pkg/manager/manager_tasks.go | 115 +++++++++++++++--- pkg/manager/task_generate_markers.go | 9 +- .../SettingsTasksPanel/SettingsTasksPanel.tsx | 91 +++++++++++++- ui/v2/src/core/StashService.ts | 13 ++ 13 files changed, 283 insertions(+), 68 deletions(-) delete mode 100644 pkg/manager/manager_subscription_handler.go diff --git a/graphql/documents/queries/settings/metadata.graphql b/graphql/documents/queries/settings/metadata.graphql index ef45011f9..9899f1dcd 100644 --- a/graphql/documents/queries/settings/metadata.graphql +++ b/graphql/documents/queries/settings/metadata.graphql @@ -16,4 +16,16 @@ query MetadataGenerate($input: GenerateMetadataInput!) { query MetadataClean { metadataClean +} + +query JobStatus { + jobStatus { + progress + status + message + } +} + +query StopJob { + stopJob } \ No newline at end of file diff --git a/graphql/documents/subscriptions.graphql b/graphql/documents/subscriptions.graphql index f9673dd8e..a4c367ab1 100644 --- a/graphql/documents/subscriptions.graphql +++ b/graphql/documents/subscriptions.graphql @@ -1,5 +1,9 @@ subscription MetadataUpdate { - metadataUpdate + metadataUpdate { + progress + status + message + } } subscription LoggingSubscribe { diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index 19f2d5e91..c03fc6df4 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -67,6 +67,9 @@ type Query { """Clean metadata. Returns the job ID""" metadataClean: String! + jobStatus: MetadataUpdateStatus! + stopJob: Boolean! + # Get everything allPerformers: [Performer!]! @@ -106,7 +109,7 @@ type Mutation { type Subscription { """Update from the metadata manager""" - metadataUpdate: String! + metadataUpdate: MetadataUpdateStatus! loggingSubscribe: [LogEntry!]! } diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index 28190c487..025d3e11f 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -7,4 +7,10 @@ input GenerateMetadataInput { input ScanMetadataInput { nameFromMetadata: Boolean! +} + +type MetadataUpdateStatus { + progress: Float! + status: String! + message: String! } \ No newline at end of file diff --git a/pkg/api/resolver_query_metadata.go b/pkg/api/resolver_query_metadata.go index 3dfea9698..8acb708ae 100644 --- a/pkg/api/resolver_query_metadata.go +++ b/pkg/api/resolver_query_metadata.go @@ -31,3 +31,18 @@ func (r *queryResolver) MetadataClean(ctx context.Context) (string, error) { manager.GetInstance().Clean() return "todo", nil } + +func (r *queryResolver) JobStatus(ctx context.Context) (*models.MetadataUpdateStatus, error) { + status := manager.GetInstance().Status + ret := models.MetadataUpdateStatus{ + Progress: status.Progress, + Status: status.Status.String(), + Message: "", + } + + return &ret, nil +} + +func (r *queryResolver) StopJob(ctx context.Context) (bool, error) { + return manager.GetInstance().Status.Stop(), nil +} diff --git a/pkg/api/resolver_subscription_metadata.go b/pkg/api/resolver_subscription_metadata.go index 628197c9e..b30b83892 100644 --- a/pkg/api/resolver_subscription_metadata.go +++ b/pkg/api/resolver_subscription_metadata.go @@ -2,20 +2,32 @@ package api import ( "context" - "github.com/stashapp/stash/pkg/manager" "time" + + "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/pkg/models" ) -func (r *subscriptionResolver) MetadataUpdate(ctx context.Context) (<-chan string, error) { - msg := make(chan string, 1) +func (r *subscriptionResolver) MetadataUpdate(ctx context.Context) (<-chan *models.MetadataUpdateStatus, error) { + msg := make(chan *models.MetadataUpdateStatus, 1) ticker := time.NewTicker(5 * time.Second) go func() { + lastStatus := manager.TaskStatus{} for { select { case _ = <-ticker.C: - manager.GetInstance().HandleMetadataUpdateSubscriptionTick(msg) + thisStatus := manager.GetInstance().Status + if thisStatus != lastStatus { + ret := models.MetadataUpdateStatus{ + Progress: thisStatus.Progress, + Status: thisStatus.Status.String(), + Message: "", + } + msg <- &ret + } + lastStatus = thisStatus case <-ctx.Done(): ticker.Stop() close(msg) diff --git a/pkg/manager/job_status.go b/pkg/manager/job_status.go index eee425512..f412c8dc6 100644 --- a/pkg/manager/job_status.go +++ b/pkg/manager/job_status.go @@ -11,3 +11,22 @@ const ( Clean JobStatus = 5 Scrape JobStatus = 6 ) + +func (s JobStatus) String() string { + statusMessage := "" + + switch s { + case Idle: + statusMessage = "Idle" + case Import: + statusMessage = "Import" + case Export: + statusMessage = "Export" + case Scan: + statusMessage = "Scan" + case Generate: + statusMessage = "Generate" + } + + return statusMessage +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index bb21d76ca..c63a048e3 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -14,7 +14,7 @@ import ( ) type singleton struct { - Status JobStatus + Status TaskStatus Paths *paths.Paths JSON *jsonUtils @@ -38,7 +38,7 @@ func Initialize() *singleton { initFlags() initEnvs() instance = &singleton{ - Status: Idle, + Status: TaskStatus{Status: Idle, Progress: -1}, Paths: paths.NewPaths(), JSON: &jsonUtils{}, } diff --git a/pkg/manager/manager_subscription_handler.go b/pkg/manager/manager_subscription_handler.go deleted file mode 100644 index f5da02b1e..000000000 --- a/pkg/manager/manager_subscription_handler.go +++ /dev/null @@ -1,36 +0,0 @@ -package manager - -import ( - "encoding/json" - "github.com/stashapp/stash/pkg/logger" -) - -type metadataUpdatePayload struct { - Progress float64 `json:"progress"` - Message string `json:"message"` - Logs []logger.LogItem `json:"logs"` -} - -func (s *singleton) HandleMetadataUpdateSubscriptionTick(msg chan string) { - var statusMessage string - switch instance.Status { - case Idle: - statusMessage = "Idle" - case Import: - statusMessage = "Import" - case Export: - statusMessage = "Export" - case Scan: - statusMessage = "Scan" - case Generate: - statusMessage = "Generate" - } - payload := &metadataUpdatePayload{ - Progress: 0, // TODO - Message: statusMessage, - Logs: logger.LogCache, - } - payloadJSON, _ := json.Marshal(payload) - - msg <- string(payloadJSON) -} diff --git a/pkg/manager/manager_tasks.go b/pkg/manager/manager_tasks.go index 877b54c1b..c61048487 100644 --- a/pkg/manager/manager_tasks.go +++ b/pkg/manager/manager_tasks.go @@ -3,6 +3,7 @@ package manager import ( "path/filepath" "sync" + "time" "github.com/bmatcuk/doublestar" "github.com/stashapp/stash/pkg/logger" @@ -11,11 +12,47 @@ import ( "github.com/stashapp/stash/pkg/utils" ) +type TaskStatus struct { + Status JobStatus + Progress float64 + LastUpdate time.Time + stopping bool +} + +func (t *TaskStatus) Stop() bool { + t.stopping = true + t.updated() + return true +} + +func (t *TaskStatus) SetStatus(s JobStatus) { + t.Status = s + t.updated() +} + +func (t *TaskStatus) setProgress(upTo int, total int) { + if total == 0 { + t.Progress = 1 + } + t.Progress = float64(upTo) / float64(total) + t.updated() +} + +func (t *TaskStatus) indefiniteProgress() { + t.Progress = -1 + t.updated() +} + +func (t *TaskStatus) updated() { + t.LastUpdate = time.Now() +} + func (s *singleton) Scan(nameFromMetadata bool) { - if s.Status != Idle { + if s.Status.Status != Idle { return } - s.Status = Scan + s.Status.SetStatus(Scan) + s.Status.indefiniteProgress() go func() { defer s.returnToIdleState() @@ -26,10 +63,23 @@ func (s *singleton) Scan(nameFromMetadata bool) { globResults, _ := doublestar.Glob(globPath) results = append(results, globResults...) } - logger.Infof("Starting scan of %d files", len(results)) + + if s.Status.stopping { + logger.Info("Stopping due to user request") + return + } + + total := len(results) + logger.Infof("Starting scan of %d files", total) var wg sync.WaitGroup - for _, path := range results { + s.Status.Progress = 0 + for i, path := range results { + s.Status.setProgress(i, total) + if s.Status.stopping { + logger.Info("Stopping due to user request") + return + } wg.Add(1) task := ScanTask{FilePath: path, NameFromMetadata: nameFromMetadata} go task.Start(&wg) @@ -41,10 +91,11 @@ func (s *singleton) Scan(nameFromMetadata bool) { } func (s *singleton) Import() { - if s.Status != Idle { + if s.Status.Status != Idle { return } - s.Status = Import + s.Status.SetStatus(Import) + s.Status.indefiniteProgress() go func() { defer s.returnToIdleState() @@ -58,10 +109,11 @@ func (s *singleton) Import() { } func (s *singleton) Export() { - if s.Status != Idle { + if s.Status.Status != Idle { return } - s.Status = Export + s.Status.SetStatus(Export) + s.Status.indefiniteProgress() go func() { defer s.returnToIdleState() @@ -75,10 +127,11 @@ func (s *singleton) Export() { } func (s *singleton) Generate(sprites bool, previews bool, markers bool, transcodes bool) { - if s.Status != Idle { + if s.Status.Status != Idle { return } - s.Status = Generate + s.Status.SetStatus(Generate) + s.Status.indefiniteProgress() qb := models.NewSceneQueryBuilder() //this.job.total = await ObjectionUtils.getCount(Scene); @@ -95,7 +148,21 @@ func (s *singleton) Generate(sprites bool, previews bool, markers bool, transcod delta := utils.Btoi(sprites) + utils.Btoi(previews) + utils.Btoi(markers) + utils.Btoi(transcodes) var wg sync.WaitGroup - for _, scene := range scenes { + s.Status.Progress = 0 + total := len(scenes) + + if s.Status.stopping { + logger.Info("Stopping due to user request") + return + } + + for i, scene := range scenes { + s.Status.setProgress(i, total) + if s.Status.stopping { + logger.Info("Stopping due to user request") + return + } + if scene == nil { logger.Errorf("nil scene, skipping generate") continue @@ -134,10 +201,11 @@ func (s *singleton) Generate(sprites bool, previews bool, markers bool, transcod } func (s *singleton) Clean() { - if s.Status != Idle { + if s.Status.Status != Idle { return } - s.Status = Clean + s.Status.SetStatus(Clean) + s.Status.indefiniteProgress() qb := models.NewSceneQueryBuilder() go func() { @@ -150,8 +218,21 @@ func (s *singleton) Clean() { return } + if s.Status.stopping { + logger.Info("Stopping due to user request") + return + } + var wg sync.WaitGroup - for _, scene := range scenes { + s.Status.Progress = 0 + total := len(scenes) + for i, scene := range scenes { + s.Status.setProgress(i, total) + if s.Status.stopping { + logger.Info("Stopping due to user request") + return + } + if scene == nil { logger.Errorf("nil scene, skipping generate") continue @@ -173,8 +254,10 @@ func (s *singleton) returnToIdleState() { logger.Info("recovered from ", r) } - if s.Status == Generate { + if s.Status.Status == Generate { instance.Paths.Generated.RemoveTmpDir() } - s.Status = Idle + s.Status.SetStatus(Idle) + s.Status.indefiniteProgress() + s.Status.stopping = false } diff --git a/pkg/manager/task_generate_markers.go b/pkg/manager/task_generate_markers.go index 5b67007cd..692c078c5 100644 --- a/pkg/manager/task_generate_markers.go +++ b/pkg/manager/task_generate_markers.go @@ -1,14 +1,15 @@ package manager import ( - "github.com/stashapp/stash/pkg/ffmpeg" - "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" "os" "path/filepath" "strconv" "sync" + + "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" ) type GenerateMarkersTask struct { diff --git a/ui/v2/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx b/ui/v2/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx index e10b57189..5ef1c2fe7 100644 --- a/ui/v2/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx +++ b/ui/v2/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx @@ -6,8 +6,10 @@ import { FormGroup, H4, AnchorButton, + ProgressBar, + H5, } from "@blueprintjs/core"; -import React, { FunctionComponent, useState } from "react"; +import React, { FunctionComponent, useState, useEffect } from "react"; import { StashService } from "../../../core/StashService"; import { ErrorUtils } from "../../../utils/errors"; import { ToastUtils } from "../../../utils/toasts"; @@ -20,10 +22,58 @@ export const SettingsTasksPanel: FunctionComponent = (props: IProps) => const [isImportAlertOpen, setIsImportAlertOpen] = useState(false); const [isCleanAlertOpen, setIsCleanAlertOpen] = useState(false); const [nameFromMetadata, setNameFromMetadata] = useState(true); + const [status, setStatus] = useState(""); + const [progress, setProgress] = useState(undefined); + + const jobStatus = StashService.useJobStatus(); + const metadataUpdate = StashService.useMetadataUpdate(); + + function statusToText(status : string) { + switch(status) { + case "Idle": + return "Idle"; + case "Scan": + return "Scanning for new content"; + case "Generate": + return "Generating supporting files"; + case "Clean": + return "Cleaning the database"; + case "Export": + return "Exporting to JSON"; + case "Import": + return "Importing from JSON"; + } + + return "Idle"; + } + + useEffect(() => { + if (!!jobStatus.data && !!jobStatus.data.jobStatus) { + setStatus(statusToText(jobStatus.data.jobStatus.status)); + var newProgress = jobStatus.data.jobStatus.progress; + if (newProgress < 0) { + setProgress(undefined); + } else { + setProgress(newProgress); + } + } + }, [jobStatus.data]); + + useEffect(() => { + if (!!metadataUpdate.data && !!metadataUpdate.data.metadataUpdate) { + setStatus(statusToText(metadataUpdate.data.metadataUpdate.status)); + var newProgress = metadataUpdate.data.metadataUpdate.progress; + if (newProgress < 0) { + setProgress(undefined); + } else { + setProgress(newProgress); + } + } + }, [metadataUpdate.data]); function onImport() { setIsImportAlertOpen(false); - StashService.queryMetadataImport(); + StashService.queryMetadataImport().then(() => { jobStatus.refetch()}); } function renderImportAlert() { @@ -47,7 +97,7 @@ export const SettingsTasksPanel: FunctionComponent = (props: IProps) => function onClean() { setIsCleanAlertOpen(false); - StashService.queryMetadataClean(); + StashService.queryMetadataClean().then(() => { jobStatus.refetch()}); } function renderCleanAlert() { @@ -74,16 +124,49 @@ export const SettingsTasksPanel: FunctionComponent = (props: IProps) => try { await StashService.queryMetadataScan({nameFromMetadata}); ToastUtils.success("Started scan"); + jobStatus.refetch(); } catch (e) { ErrorUtils.handle(e); } } + function maybeRenderStop() { + if (!status || status === "Idle") { + return undefined; + } + + return ( + <> + +