From 0cd9a0a474cb741ca29dc975de6e255f7fedc62d Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Thu, 24 Mar 2022 09:22:41 +1100 Subject: [PATCH] Python path setting (#2409) * Add python package * Add python path backend config * Add python path to system settings page * Apply python path to script scrapers and plugins --- graphql/documents/data/config.graphql | 1 + graphql/schema/types/config.graphql | 4 ++ internal/api/resolver_mutation_configure.go | 4 ++ internal/api/resolver_query_configuration.go | 1 + internal/manager/config/config.go | 6 +++ .../manager/config/config_concurrency_test.go | 1 + pkg/exec/command.go | 12 ++++- pkg/plugin/plugins.go | 21 +++++---- pkg/plugin/raw.go | 37 ++++++++-------- pkg/plugin/task.go | 9 ++-- pkg/python/exec.go | 44 +++++++++++++++++++ pkg/scraper/cache.go | 1 + pkg/scraper/script.go | 40 ++++++++--------- pkg/scraper/xpath_test.go | 4 ++ .../components/Changelog/versions/v0140.md | 3 ++ .../Settings/SettingsSystemPanel.tsx | 8 ++++ ui/v2.5/src/locales/en-GB.json | 4 ++ 17 files changed, 147 insertions(+), 53 deletions(-) create mode 100644 pkg/python/exec.go diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index a4b9ff9cb..396526414 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -44,6 +44,7 @@ fragment ConfigGeneralData on ConfigGeneralResult { endpoint api_key } + pythonPath } fragment ConfigInterfaceData on ConfigInterfaceResult { diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index f2f98f393..f27870b36 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -107,6 +107,8 @@ input ConfigGeneralInput { scraperCertCheck: Boolean @deprecated(reason: "use mutation ConfigureScraping(input: ConfigScrapingInput) instead") """Stash-box instances used for tagging""" stashBoxes: [StashBoxInput!] + """Python path - resolved using path if unset""" + pythonPath: String } type ConfigGeneralResult { @@ -188,6 +190,8 @@ type ConfigGeneralResult { scraperCertCheck: Boolean! @deprecated(reason: "use ConfigResult.scraping instead") """Stash-box instances used for tagging""" stashBoxes: [StashBox!]! + """Python path - resolved using path if unset""" + pythonPath: String! } input ConfigDisableDropdownCreateInput { diff --git a/internal/api/resolver_mutation_configure.go b/internal/api/resolver_mutation_configure.go index 7ac05941f..982dbc462 100644 --- a/internal/api/resolver_mutation_configure.go +++ b/internal/api/resolver_mutation_configure.go @@ -265,6 +265,10 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co c.Set(config.StashBoxes, input.StashBoxes) } + if input.PythonPath != nil { + c.Set(config.PythonPath, input.PythonPath) + } + if err := c.Write(); err != nil { return makeConfigGeneralResult(), err } diff --git a/internal/api/resolver_query_configuration.go b/internal/api/resolver_query_configuration.go index 1b91a5f0a..ad0a2c142 100644 --- a/internal/api/resolver_query_configuration.go +++ b/internal/api/resolver_query_configuration.go @@ -120,6 +120,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult { ScraperCertCheck: config.GetScraperCertCheck(), ScraperCDPPath: &scraperCDPPath, StashBoxes: config.GetStashBoxes(), + PythonPath: config.GetPythonPath(), } } diff --git a/internal/manager/config/config.go b/internal/manager/config/config.go index 818b53192..bd1235cd5 100644 --- a/internal/manager/config/config.go +++ b/internal/manager/config/config.go @@ -105,6 +105,8 @@ const ( // stash-box options StashBoxes = "stash_boxes" + PythonPath = "python_path" + // plugin options PluginsPath = "plugins_path" @@ -624,6 +626,10 @@ func (i *Instance) GetPluginsPath() string { return i.getString(PluginsPath) } +func (i *Instance) GetPythonPath() string { + return i.getString(PythonPath) +} + func (i *Instance) GetHost() string { ret := i.getString(Host) if ret == "" { diff --git a/internal/manager/config/config_concurrency_test.go b/internal/manager/config/config_concurrency_test.go index d3e0c1545..8af130419 100644 --- a/internal/manager/config/config_concurrency_test.go +++ b/internal/manager/config/config_concurrency_test.go @@ -108,6 +108,7 @@ func TestConcurrentConfigAccess(t *testing.T) { i.SetChecksumDefaultValues(i.GetVideoFileNamingAlgorithm(), i.IsCalculateMD5()) i.Set(AutostartVideoOnPlaySelected, i.GetAutostartVideoOnPlaySelected()) i.Set(ContinuePlaylistDefault, i.GetContinuePlaylistDefault()) + i.Set(PythonPath, i.GetPythonPath()) } wg.Done() }(k) diff --git a/pkg/exec/command.go b/pkg/exec/command.go index 366de9215..bf86f6fed 100644 --- a/pkg/exec/command.go +++ b/pkg/exec/command.go @@ -1,7 +1,10 @@ // Package exec provides functions that wrap os/exec functions. These functions prevent external commands from opening windows on the Windows platform. package exec -import "os/exec" +import ( + "context" + "os/exec" +) // Command wraps the exec.Command function, preventing Windows from opening a window when starting. func Command(name string, arg ...string) *exec.Cmd { @@ -9,3 +12,10 @@ func Command(name string, arg ...string) *exec.Cmd { hideExecShell(ret) return ret } + +// CommandContext wraps the exec.CommandContext function, preventing Windows from opening a window when starting. +func CommandContext(ctx context.Context, name string, arg ...string) *exec.Cmd { + ret := exec.CommandContext(ctx, name, arg...) + hideExecShell(ret) + return ret +} diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 4f635b150..85fde229b 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -28,6 +28,7 @@ type ServerConfig interface { GetConfigPath() string HasTLSConfig() bool GetPluginsPath() string + GetPythonPath() string } // Cache stores plugin details. @@ -167,11 +168,12 @@ func (c Cache) CreateTask(ctx context.Context, pluginID string, operationName st } task := pluginTask{ - plugin: plugin, - operation: operation, - input: buildPluginInput(plugin, operation, serverConnection, args), - progress: progress, - gqlHandler: c.gqlHandler, + plugin: plugin, + operation: operation, + input: buildPluginInput(plugin, operation, serverConnection, args), + progress: progress, + gqlHandler: c.gqlHandler, + serverConfig: c.config, } return task.createTask(), nil } @@ -216,10 +218,11 @@ func (c Cache) executePostHooks(ctx context.Context, hookType HookTriggerEnum, h addHookContext(pluginInput.Args, hookContext) pt := pluginTask{ - plugin: &p, - operation: &h.OperationConfig, - input: pluginInput, - gqlHandler: c.gqlHandler, + plugin: &p, + operation: &h.OperationConfig, + input: pluginInput, + gqlHandler: c.gqlHandler, + serverConfig: c.config, } task := pt.createTask() diff --git a/pkg/plugin/raw.go b/pkg/plugin/raw.go index 48f3064d7..95ffa46f3 100644 --- a/pkg/plugin/raw.go +++ b/pkg/plugin/raw.go @@ -1,6 +1,7 @@ package plugin import ( + "context" "encoding/json" "errors" "fmt" @@ -11,6 +12,7 @@ import ( stashExec "github.com/stashapp/stash/pkg/exec" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/plugin/common" + "github.com/stashapp/stash/pkg/python" ) type rawTaskBuilder struct{} @@ -30,19 +32,6 @@ type rawPluginTask struct { done chan bool } -func FindPythonExecutable() (string, error) { - _, err := exec.LookPath("python3") - - if err != nil { - _, err = exec.LookPath("python") - if err != nil { - return "", err - } - return "python", nil - } - return "python3", nil -} - func (t *rawPluginTask) Start() error { if t.started { return errors.New("task already started") @@ -53,14 +42,26 @@ func (t *rawPluginTask) Start() error { return fmt.Errorf("empty exec value in operation %s", t.operation.Name) } - if command[0] == "python" || command[0] == "python3" { - executable, err := FindPythonExecutable() - if err == nil { - command[0] = executable + var cmd *exec.Cmd + if python.IsPythonCommand(command[0]) { + pythonPath := t.serverConfig.GetPythonPath() + var p *python.Python + if pythonPath != "" { + p = python.New(pythonPath) + } else { + p, _ = python.Resolve() } + + if p != nil { + cmd = p.Command(context.TODO(), command[1:]) + } + + // if could not find python, just use the command args as-is } - cmd := stashExec.Command(command[0], command[1:]...) + if cmd == nil { + cmd = stashExec.Command(command[0], command[1:]...) + } stdin, err := cmd.StdinPipe() if err != nil { diff --git a/pkg/plugin/task.go b/pkg/plugin/task.go index 4b4a9d870..e80c96ff7 100644 --- a/pkg/plugin/task.go +++ b/pkg/plugin/task.go @@ -30,10 +30,11 @@ type taskBuilder interface { } type pluginTask struct { - plugin *Config - operation *OperationConfig - input common.PluginInput - gqlHandler http.Handler + plugin *Config + operation *OperationConfig + input common.PluginInput + gqlHandler http.Handler + serverConfig ServerConfig progress chan float64 result *common.PluginOutput diff --git a/pkg/python/exec.go b/pkg/python/exec.go new file mode 100644 index 000000000..bd5679533 --- /dev/null +++ b/pkg/python/exec.go @@ -0,0 +1,44 @@ +package python + +import ( + "context" + "os/exec" + + stashExec "github.com/stashapp/stash/pkg/exec" +) + +type Python string + +func (p *Python) Command(ctx context.Context, args []string) *exec.Cmd { + return stashExec.CommandContext(ctx, string(*p), args...) +} + +// New returns a new Python instance at the given path. +func New(path string) *Python { + ret := Python(path) + return &ret +} + +// Resolve tries to find the python executable in the system. +// It first checks for python3, then python. +// Returns nil and an exec.ErrNotFound error if not found. +func Resolve() (*Python, error) { + _, err := exec.LookPath("python3") + + if err != nil { + _, err = exec.LookPath("python") + if err != nil { + return nil, err + } + ret := Python("python") + return &ret, nil + } + + ret := Python("python3") + return &ret, nil +} + +// IsPythonCommand returns true if arg is "python" or "python3" +func IsPythonCommand(arg string) bool { + return arg == "python" || arg == "python3" +} diff --git a/pkg/scraper/cache.go b/pkg/scraper/cache.go index 4b1d67d35..2190dcb03 100644 --- a/pkg/scraper/cache.go +++ b/pkg/scraper/cache.go @@ -36,6 +36,7 @@ type GlobalConfig interface { GetScrapersPath() string GetScraperCDPPath() string GetScraperCertCheck() bool + GetPythonPath() string } func isCDPPathHTTP(c GlobalConfig) bool { diff --git a/pkg/scraper/script.go b/pkg/scraper/script.go index d182b1ee3..e2847b9cd 100644 --- a/pkg/scraper/script.go +++ b/pkg/scraper/script.go @@ -13,6 +13,7 @@ import ( stashExec "github.com/stashapp/stash/pkg/exec" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/python" ) var ErrScraperScript = errors.New("scraper script error") @@ -34,14 +35,27 @@ func newScriptScraper(scraper scraperTypeConfig, config config, globalConfig Glo func (s *scriptScraper) runScraperScript(inString string, out interface{}) error { command := s.scraper.Script - if command[0] == "python" || command[0] == "python3" { - executable, err := findPythonExecutable() - if err == nil { - command[0] = executable + var cmd *exec.Cmd + if python.IsPythonCommand(command[0]) { + pythonPath := s.globalConfig.GetPythonPath() + var p *python.Python + if pythonPath != "" { + p = python.New(pythonPath) + } else { + p, _ = python.Resolve() } + + if p != nil { + cmd = p.Command(context.TODO(), command[1:]) + } + + // if could not find python, just use the command args as-is + } + + if cmd == nil { + cmd = stashExec.Command(command[0], command[1:]...) } - cmd := stashExec.Command(command[0], command[1:]...) cmd.Dir = filepath.Dir(s.config.path) stdin, err := cmd.StdinPipe() @@ -220,22 +234,6 @@ func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mod return ret, err } -func findPythonExecutable() (string, error) { - _, err := exec.LookPath("python3") - - if err != nil { - _, err = exec.LookPath("python") - - if err != nil { - return "", err - } - - return "python", nil - } - - return "python3", nil -} - func handleScraperStderr(name string, scraperOutputReader io.ReadCloser) { const scraperPrefix = "[Scrape / %s] " diff --git a/pkg/scraper/xpath_test.go b/pkg/scraper/xpath_test.go index 7f0ab25d5..782b753f0 100644 --- a/pkg/scraper/xpath_test.go +++ b/pkg/scraper/xpath_test.go @@ -830,6 +830,10 @@ func (mockGlobalConfig) GetScraperExcludeTagPatterns() []string { return nil } +func (mockGlobalConfig) GetPythonPath() string { + return "" +} + func TestSubScrape(t *testing.T) { retHTML := `
diff --git a/ui/v2.5/src/components/Changelog/versions/v0140.md b/ui/v2.5/src/components/Changelog/versions/v0140.md index 94fdf6227..1c3e2a2a8 100644 --- a/ui/v2.5/src/components/Changelog/versions/v0140.md +++ b/ui/v2.5/src/components/Changelog/versions/v0140.md @@ -1,3 +1,6 @@ +### ✨ New Features +* Add python location in System Settings for script scrapers and plugins. ([#2409](https://github.com/stashapp/stash/pull/2409)) + ### 🎨 Improvements * Maintain lightbox settings and add lightbox settings to Interface settings page. ([#2406](https://github.com/stashapp/stash/pull/2406)) * Image lightbox now transitions to next/previous image when scrolling in pan-Y mode. ([#2403](https://github.com/stashapp/stash/pull/2403)) diff --git a/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx b/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx index 7190afa98..66de2513f 100644 --- a/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx @@ -147,6 +147,14 @@ export const SettingsConfigurationPanel: React.FC = () => { value={general.customPerformerImageLocation ?? undefined} onChange={(v) => saveGeneral({ customPerformerImageLocation: v })} /> + + saveGeneral({ pythonPath: v })} + /> diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index 95f95c44d..7442c586b 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -279,6 +279,10 @@ "number_of_parallel_task_for_scan_generation_head": "Number of parallel task for scan/generation", "parallel_scan_head": "Parallel Scan/Generation", "preview_generation": "Preview Generation", + "python_path": { + "description": "Location of python executable. Used for script scrapers and plugins. If blank, python will be resolved from the environment", + "heading": "Python Path" + }, "scraper_user_agent": "Scraper User Agent", "scraper_user_agent_desc": "User-Agent string used during scrape http requests", "scrapers_path": {