Add plugin tasks (#651)

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/go-chi/chi v4.0.2+incompatible
 	github.com/gobuffalo/packr/v2 v2.0.2
 	github.com/golang-migrate/migrate/v4 v4.3.1
+	github.com/gorilla/securecookie v1.1.1
 	github.com/gorilla/sessions v1.2.0
 	github.com/gorilla/websocket v1.4.0
 	github.com/h2non/filetype v1.0.8
@@ -18,6 +19,7 @@ require (
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/json-iterator/go v1.1.9
 	github.com/mattn/go-sqlite3 v1.13.0
+	github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007
 	github.com/rs/cors v1.6.0
 	github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f
 	github.com/sirupsen/logrus v1.4.2

go.sum

@@ -486,6 +486,8 @@ github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5EL
 github.com/monoculum/formam v0.0.0-20180901015400-4e68be1d79ba/go.mod h1:RKgILGEJq24YyJ2ban8EO0RUVSJlF1pGsEvoLEACr/Q=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA=
+github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007 h1:Ohgj9L0EYOgXxkDp+bczlMBiulwmqYzQpvQNUdtt3oc=
+github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007/go.mod h1:wKCOWMb6iNlvKiOToY2cNuaovSXvIiv1zDi9QDR7aGQ=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=

graphql/documents/mutations/plugins.graphql

@@ -0,0 +1,7 @@
+mutation ReloadPlugins {
+  reloadPlugins
+}
+
+mutation RunPluginTask($plugin_id: ID!, $task_name: String!, $args: [PluginArgInput!]) {
+  runPluginTask(plugin_id: $plugin_id, task_name: $task_name, args: $args)
+}

graphql/documents/queries/plugins.graphql

@@ -0,0 +1,25 @@
+query Plugins {
+  plugins {
+    id
+    name
+    description
+    url
+    version
+
+    tasks {
+      name
+      description
+    }
+  }
+}
+
+query PluginTasks {
+  pluginTasks {
+    name
+    description
+    plugin {
+      id
+      name
+    }
+  }
+}

graphql/schema/schema.graphql

@@ -76,6 +76,13 @@ type Query {
   """Scrape a list of performers from a query"""
   scrapeFreeonesPerformerList(query: String!): [String!]!
 
+  # Plugins
+  """List loaded plugins"""
+  plugins: [Plugin!]
+  """List available plugin operations"""
+  pluginTasks: [PluginTask!]
+
+
   # Config
   """Returns the current, complete configuration"""
   configuration: ConfigResult!
@@ -166,6 +173,10 @@ type Mutation {
   """Reload scrapers"""
   reloadScrapers: Boolean!
 
+  """Run plugin task. Returns the job ID"""
+  runPluginTask(plugin_id: ID!, task_name: String!, args: [PluginArgInput!]): String!
+  reloadPlugins: Boolean!
+
   stopJob: Boolean!
 }
 

graphql/schema/types/plugin.graphql

@@ -0,0 +1,35 @@
+
+type Plugin {
+    id: ID!
+    name: String!
+    description: String
+    url: String
+    version: String
+
+    tasks: [PluginTask!]
+}
+
+type PluginTask {
+    name: String!
+    description: String
+    plugin: Plugin!
+}
+
+type PluginResult {
+    error: String
+    result: String
+}
+
+input PluginArgInput {
+    key: String!
+    value: PluginValueInput
+}
+
+input PluginValueInput {
+    str: String
+    i: Int
+    b: Boolean
+    f: Float
+    o: [PluginArgInput!]
+    a: [PluginValueInput!]
+}

pkg/api/resolver_mutation_plugin.go

@@ -0,0 +1,48 @@
+package api
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/manager"
+	"github.com/stashapp/stash/pkg/manager/config"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/plugin/common"
+)
+
+func (r *mutationResolver) RunPluginTask(ctx context.Context, pluginID string, taskName string, args []*models.PluginArgInput) (string, error) {
+	currentUser := getCurrentUserID(ctx)
+
+	var cookie *http.Cookie
+	var err error
+	if currentUser != nil {
+		cookie, err = createSessionCookie(*currentUser)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	serverConnection := common.StashServerConnection{
+		Scheme:        "http",
+		Port:          config.GetPort(),
+		SessionCookie: cookie,
+		Dir:           config.GetConfigPath(),
+	}
+
+	if HasTLSConfig() {
+		serverConnection.Scheme = "https"
+	}
+
+	manager.GetInstance().RunPluginTask(pluginID, taskName, args, serverConnection)
+	return "todo", nil
+}
+
+func (r *mutationResolver) ReloadPlugins(ctx context.Context) (bool, error) {
+	err := manager.GetInstance().PluginCache.ReloadPlugins()
+	if err != nil {
+		logger.Errorf("Error reading plugin configs: %s", err.Error())
+	}
+
+	return true, nil
+}

pkg/api/resolver_query_plugin.go

@@ -0,0 +1,16 @@
+package api
+
+import (
+	"context"
+
+	"github.com/stashapp/stash/pkg/manager"
+	"github.com/stashapp/stash/pkg/models"
+)
+
+func (r *queryResolver) Plugins(ctx context.Context) ([]*models.Plugin, error) {
+	return manager.GetInstance().PluginCache.ListPlugins(), nil
+}
+
+func (r *queryResolver) PluginTasks(ctx context.Context) ([]*models.PluginTask, error) {
+	return manager.GetInstance().PluginCache.ListPluginTasks(), nil
+}

pkg/api/server.go

@@ -357,6 +357,15 @@ func makeTLSConfig() *tls.Config {
 	return tlsConfig
 }
 
+func HasTLSConfig() bool {
+	ret, _ := utils.FileExists(paths.GetSSLCert())
+	if ret {
+		ret, _ = utils.FileExists(paths.GetSSLKey())
+	}
+
+	return ret
+}
+
 type contextKey struct {
 	name string
 }

pkg/api/session.go

@@ -1,12 +1,14 @@
 package api
 
 import (
+	"context"
 	"fmt"
 	"html/template"
 	"net/http"
 
 	"github.com/stashapp/stash/pkg/manager/config"
 
+	"github.com/gorilla/securecookie"
 	"github.com/gorilla/sessions"
 )
 
@@ -125,3 +127,26 @@ func getSessionUserID(w http.ResponseWriter, r *http.Request) (string, error) {
 
 	return "", nil
 }
+
+func getCurrentUserID(ctx context.Context) *string {
+	userCtxVal := ctx.Value(ContextUser)
+	if userCtxVal != nil {
+		currentUser := userCtxVal.(string)
+		return &currentUser
+	}
+
+	return nil
+}
+
+func createSessionCookie(username string) (*http.Cookie, error) {
+	session := sessions.NewSession(sessionStore, cookieName)
+	session.Values[userIDKey] = username
+
+	encoded, err := securecookie.EncodeMulti(session.Name(), session.Values,
+		sessionStore.Codecs...)
+	if err != nil {
+		return nil, err
+	}
+
+	return sessions.NewCookie(session.Name(), encoded, session.Options), nil
+}

pkg/manager/config/config.go

@@ -67,6 +67,9 @@ const ScrapersPath = "scrapers_path"
 const ScraperUserAgent = "scraper_user_agent"
 const ScraperCDPPath = "scraper_cdp_path"
 
+// plugin options
+const PluginsPath = "plugins_path"
+
 // i18n
 const Language = "language"
 
@@ -106,6 +109,11 @@ func Write() error {
 	return viper.WriteConfig()
 }
 
+func GetConfigPath() string {
+	configFileUsed := viper.ConfigFileUsed()
+	return filepath.Dir(configFileUsed)
+}
+
 func GetStashPaths() []string {
 	return viper.GetStringSlice(Stash)
 }
@@ -136,10 +144,8 @@ func GetSessionStoreKey() []byte {
 
 func GetDefaultScrapersPath() string {
 	// default to the same directory as the config file
-	configFileUsed := viper.ConfigFileUsed()
-	configDir := filepath.Dir(configFileUsed)
 
-	fn := filepath.Join(configDir, "scrapers")
+	fn := filepath.Join(GetConfigPath(), "scrapers")
 
 	return fn
 }
@@ -192,6 +198,17 @@ func GetScraperCDPPath() string {
 	return viper.GetString(ScraperCDPPath)
 }
 
+func GetDefaultPluginsPath() string {
+	// default to the same directory as the config file
+	fn := filepath.Join(GetConfigPath(), "plugins")
+
+	return fn
+}
+
+func GetPluginsPath() string {
+	return viper.GetString(PluginsPath)
+}
+
 func GetHost() string {
 	return viper.GetString(Host)
 }

pkg/manager/job_status.go

@@ -12,6 +12,7 @@ const (
 	Scrape          JobStatus = 6
 	AutoTag         JobStatus = 7
 	Migrate         JobStatus = 8
+	PluginOperation JobStatus = 9
 )
 
 func (s JobStatus) String() string {
@@ -34,6 +35,8 @@ func (s JobStatus) String() string {
 		statusMessage = "Migrate"
 	case Clean:
 		statusMessage = "Clean"
+	case PluginOperation:
+		statusMessage = "Plugin Operation"
 	}
 
 	return statusMessage

pkg/manager/manager.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/manager/config"
 	"github.com/stashapp/stash/pkg/manager/paths"
+	"github.com/stashapp/stash/pkg/plugin"
 	"github.com/stashapp/stash/pkg/scraper"
 	"github.com/stashapp/stash/pkg/utils"
 )
@@ -22,6 +23,7 @@ type singleton struct {
 	FFMPEGPath  string
 	FFProbePath string
 
+	PluginCache  *plugin.Cache
 	ScraperCache *scraper.Cache
 }
 
@@ -51,6 +53,7 @@ func Initialize() *singleton {
 			Paths:  paths.NewPaths(),
 			JSON:   &jsonUtils{},
 
+			PluginCache:  initPluginCache(),
 			ScraperCache: initScraperCache(),
 		}
 
@@ -88,8 +91,9 @@ func initConfig() {
 	// Set generated to the metadata path for backwards compat
 	viper.SetDefault(config.Generated, viper.GetString(config.Metadata))
 
-	// Set default scrapers path
+	// Set default scrapers and plugins paths
 	viper.SetDefault(config.ScrapersPath, config.GetDefaultScrapersPath())
+	viper.SetDefault(config.PluginsPath, config.GetDefaultPluginsPath())
 
 	// Disabling config watching due to race condition issue
 	// See: https://github.com/spf13/viper/issues/174
@@ -153,6 +157,16 @@ func initLog() {
 	logger.Init(config.GetLogFile(), config.GetLogOut(), config.GetLogLevel())
 }
 
+func initPluginCache() *plugin.Cache {
+	ret, err := plugin.NewCache(config.GetPluginsPath())
+
+	if err != nil {
+		logger.Errorf("Error reading plugin configs: %s", err.Error())
+	}
+
+	return ret
+}
+
 // initScraperCache initializes a new scraper cache and returns it.
 func initScraperCache() *scraper.Cache {
 	scraperConfig := scraper.GlobalConfig{

pkg/manager/manager_tasks.go

@@ -65,6 +65,13 @@ func (t *TaskStatus) setProgress(upTo int, total int) {
 	t.updated()
 }
 
+func (t *TaskStatus) setProgressPercent(progress float64) {
+	if progress != t.Progress {
+		t.Progress = progress
+		t.updated()
+	}
+}
+
 func (t *TaskStatus) incrementProgress() {
 	t.setProgress(t.upTo+1, t.total)
 }

pkg/manager/task_plugin.go

@@ -0,0 +1,71 @@
+package manager
+
+import (
+	"time"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/plugin/common"
+)
+
+func (s *singleton) RunPluginTask(pluginID string, taskName string, args []*models.PluginArgInput, serverConnection common.StashServerConnection) {
+	if s.Status.Status != Idle {
+		return
+	}
+	s.Status.SetStatus(PluginOperation)
+	s.Status.indefiniteProgress()
+
+	go func() {
+		defer s.returnToIdleState()
+
+		progress := make(chan float64)
+		task, err := s.PluginCache.CreateTask(pluginID, taskName, serverConnection, args, progress)
+		if err != nil {
+			logger.Errorf("Error creating plugin task: %s", err.Error())
+			return
+		}
+
+		err = task.Start()
+		if err != nil {
+			logger.Errorf("Error running plugin task: %s", err.Error())
+			return
+		}
+
+		done := make(chan bool)
+		go func() {
+			defer close(done)
+			task.Wait()
+
+			output := task.GetResult()
+			if output == nil {
+				logger.Debug("Plugin returned no result")
+			} else {
+				if output.Error != nil {
+					logger.Errorf("Plugin returned error: %s", *output.Error)
+				} else if output.Output != nil {
+					logger.Debugf("Plugin returned: %v", output.Output)
+				}
+			}
+		}()
+
+		// TODO - refactor stop to use channels
+		// check for stop every five seconds
+		pollingTime := time.Second * 5
+		stopPoller := time.Tick(pollingTime)
+		for {
+			select {
+			case <-done:
+				return
+			case p := <-progress:
+				s.Status.setProgressPercent(p)
+			case <-stopPoller:
+				if s.Status.stopping {
+					if err := task.Stop(); err != nil {
+						logger.Errorf("Error stopping plugin operation: %s", err.Error())
+					}
+					return
+				}
+			}
+		}
+	}()
+}

pkg/plugin/args.go

@@ -0,0 +1,30 @@
+package plugin
+
+import (
+	"github.com/stashapp/stash/pkg/models"
+)
+
+func findArg(args []*models.PluginArgInput, name string) *models.PluginArgInput {
+	for _, v := range args {
+		if v.Key == name {
+			return v
+		}
+	}
+
+	return nil
+}
+
+func applyDefaultArgs(args []*models.PluginArgInput, defaultArgs map[string]string) []*models.PluginArgInput {
+	for k, v := range defaultArgs {
+		if arg := findArg(args, k); arg == nil {
+			args = append(args, &models.PluginArgInput{
+				Key: k,
+				Value: &models.PluginValueInput{
+					Str: &v,
+				},
+			})
+		}
+	}
+
+	return args
+}

pkg/plugin/common/doc.go

@@ -0,0 +1,3 @@
+// Package common encapulates data structures and functions that will be used
+// by plugin executables and the plugin subsystem in the stash server.
+package common

pkg/plugin/common/log/log.go

@@ -0,0 +1,203 @@
+// Package log provides a number of logging utility functions for encoding and
+// decoding log messages between a stash server and a plugin instance.
+//
+// Log messages sent from a plugin instance are transmitted via stderr and are
+// encoded with a prefix consisting of special character SOH, then the log
+// level (one of t, d, i, w, e, or p - corresponding to trace, debug, info,
+// warning, error and progress levels respectively), then special character
+// STX.
+//
+// The Trace, Debug, Info, Warning, and Error methods, and their equivalent
+// formatted methods are intended for use by plugin instances to transmit log
+// messages. The Progress method is also intended for sending progress data.
+//
+// Conversely, LevelFromName and DetectLogLevel are intended for use by the
+// stash server.
+package log
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"strings"
+)
+
+// Level represents a logging level for plugin outputs.
+type Level struct {
+	char byte
+	Name string
+}
+
+// Valid Level values.
+var (
+	TraceLevel = Level{
+		char: 't',
+		Name: "trace",
+	}
+	DebugLevel = Level{
+		char: 'd',
+		Name: "debug",
+	}
+	InfoLevel = Level{
+		char: 'i',
+		Name: "info",
+	}
+	WarningLevel = Level{
+		char: 'w',
+		Name: "warning",
+	}
+	ErrorLevel = Level{
+		char: 'e',
+		Name: "error",
+	}
+	ProgressLevel = Level{
+		char: 'p',
+	}
+	NoneLevel = Level{
+		Name: "none",
+	}
+)
+
+var validLevels = []Level{
+	TraceLevel,
+	DebugLevel,
+	InfoLevel,
+	WarningLevel,
+	ErrorLevel,
+	ProgressLevel,
+	NoneLevel,
+}
+
+const startLevelChar byte = 1
+const endLevelChar byte = 2
+
+func (l Level) prefix() string {
+	return string([]byte{
+		startLevelChar,
+		byte(l.char),
+		endLevelChar,
+	})
+}
+
+func (l Level) log(args ...interface{}) {
+	if l.char == 0 {
+		return
+	}
+
+	argsToUse := []interface{}{
+		l.prefix(),
+	}
+	argsToUse = append(argsToUse, args...)
+	fmt.Fprintln(os.Stderr, argsToUse...)
+}
+
+func (l Level) logf(format string, args ...interface{}) {
+	if l.char == 0 {
+		return
+	}
+
+	formatToUse := string(l.prefix()) + format + "\n"
+	fmt.Fprintf(os.Stderr, formatToUse, args...)
+}
+
+// Trace outputs a trace logging message to os.Stderr. Message is encoded with a
+// prefix that signifies to the server that it is a trace message.
+func Trace(args ...interface{}) {
+	TraceLevel.log(args...)
+}
+
+// Tracef is the equivalent of Printf outputting as a trace logging message.
+func Tracef(format string, args ...interface{}) {
+	TraceLevel.logf(format, args...)
+}
+
+// Debug outputs a debug logging message to os.Stderr. Message is encoded with a
+// prefix that signifies to the server that it is a debug message.
+func Debug(args ...interface{}) {
+	DebugLevel.log(args...)
+}
+
+// Debugf is the equivalent of Printf outputting as a debug logging message.
+func Debugf(format string, args ...interface{}) {
+	DebugLevel.logf(format, args...)
+}
+
+// Info outputs an info logging message to os.Stderr. Message is encoded with a
+// prefix that signifies to the server that it is an info message.
+func Info(args ...interface{}) {
+	InfoLevel.log(args...)
+}
+
+// Infof is the equivalent of Printf outputting as an info logging message.
+func Infof(format string, args ...interface{}) {
+	InfoLevel.logf(format, args...)
+}
+
+// Warn outputs a warning logging message to os.Stderr. Message is encoded with a
+// prefix that signifies to the server that it is a warning message.
+func Warn(args ...interface{}) {
+	WarningLevel.log(args...)
+}
+
+// Warnf is the equivalent of Printf outputting as a warning logging message.
+func Warnf(format string, args ...interface{}) {
+	WarningLevel.logf(format, args...)
+}
+
+// Error outputs an error logging message to os.Stderr. Message is encoded with a
+// prefix that signifies to the server that it is an error message.
+func Error(args ...interface{}) {
+	ErrorLevel.log(args...)
+}
+
+// Errorf is the equivalent of Printf outputting as an error logging message.
+func Errorf(format string, args ...interface{}) {
+	ErrorLevel.logf(format, args...)
+}
+
+// Progress logs the current progress value. The progress value should be
+// between 0 and 1.0 inclusively, with 1 representing that the task is
+// complete. Values outside of this range will be clamp to be within it.
+func Progress(progress float64) {
+	progress = math.Min(math.Max(0, progress), 1)
+	ProgressLevel.log(progress)
+}
+
+// LevelFromName returns the Level that matches the provided name or nil if
+// the name does not match a valid value.
+func LevelFromName(name string) *Level {
+	for _, l := range validLevels {
+		if l.Name == name {
+			return &l
+		}
+	}
+
+	return nil
+}
+
+// DetectLogLevel returns the Level and the logging string for a provided line
+// of plugin output. It parses the string for logging control characters and
+// determines the log level, if present. If not present, the plugin output
+// is returned unchanged with a nil Level.
+func DetectLogLevel(line string) (*Level, string) {
+	if len(line) < 4 || line[0] != startLevelChar || line[2] != endLevelChar {
+		return nil, line
+	}
+
+	char := line[1]
+	var level *Level
+	for _, l := range validLevels {
+		if l.char == char {
+			level = &l
+			break
+		}
+	}
+
+	if level == nil {
+		return nil, line
+	}
+
+	line = strings.TrimSpace(line[3:])
+
+	return level, line
+}

pkg/plugin/common/msg.go

@@ -0,0 +1,99 @@
+package common
+
+import "net/http"
+
+// StashServerConnection represents the connection details needed for a
+// plugin instance to connect to its parent stash server.
+type StashServerConnection struct {
+	// http or https
+	Scheme string
+
+	Port int
+
+	// Cookie for authentication purposes
+	SessionCookie *http.Cookie
+
+	// Dir specifies the directory containing the stash server's configuration
+	// file.
+	Dir string
+
+	// PluginDir specifies the directory containing the plugin configuration
+	// file.
+	PluginDir string
+}
+
+// PluginArgValue represents a single value parameter for plugin operations.
+type PluginArgValue interface{}
+
+// ArgsMap is a map of argument key to value.
+type ArgsMap map[string]PluginArgValue
+
+// String returns the string field or an empty string if the string field is
+// nil
+func (m ArgsMap) String(key string) string {
+	v, found := m[key]
+	var ret string
+	if !found {
+		return ret
+	}
+	ret, _ = v.(string)
+	return ret
+}
+
+// Int returns the int field or 0 if the int field is nil
+func (m ArgsMap) Int(key string) int {
+	v, found := m[key]
+	var ret int
+	if !found {
+		return ret
+	}
+	ret, _ = v.(int)
+	return ret
+}
+
+// Bool returns the boolean field or false if the boolean field is nil
+func (m ArgsMap) Bool(key string) bool {
+	v, found := m[key]
+	var ret bool
+	if !found {
+		return ret
+	}
+	ret, _ = v.(bool)
+	return ret
+}
+
+// Float returns the float field or 0 if the float field is nil
+func (m ArgsMap) Float(key string) float64 {
+	v, found := m[key]
+	var ret float64
+	if !found {
+		return ret
+	}
+	ret, _ = v.(float64)
+	return ret
+}
+
+// PluginInput is the data structure that is sent to plugin instances when they
+// are spawned.
+type PluginInput struct {
+	// Server details to connect to the stash server.
+	ServerConnection StashServerConnection `json:"server_connection"`
+
+	// Arguments to the plugin operation.
+	Args ArgsMap `json:"args"`
+}
+
+// PluginOutput is the data structure that is expected to be output by plugin
+// processes when execution has concluded. It is expected that this data will
+// be encoded as JSON.
+type PluginOutput struct {
+	Error  *string     `json:"error"`
+	Output interface{} `json:"output"`
+}
+
+// SetError is a convenience method that sets the Error field based on the
+// provided error.
+func (o *PluginOutput) SetError(err error) {
+	errStr := err.Error()
+	o.Error = &errStr
+}

pkg/plugin/common/rpc.go

@@ -0,0 +1,30 @@
+package common
+
+import (
+	"net/rpc/jsonrpc"
+
+	"github.com/natefinch/pie"
+)
+
+// RPCRunner is the interface that RPC plugins are expected to fulfil.
+type RPCRunner interface {
+	// Perform the operation, using the provided input and populating the
+	// output object.
+	Run(input PluginInput, output *PluginOutput) error
+
+	// Stop any running operations, if possible. No input is sent and any
+	// output is ignored.
+	Stop(input struct{}, output *bool) error
+}
+
+// ServePlugin is used by plugin instances to serve the plugin via RPC, using
+// the provided RPCRunner interface.
+func ServePlugin(iface RPCRunner) error {
+	p := pie.NewProvider()
+	if err := p.RegisterName("RPCRunner", iface); err != nil {
+		return err
+	}
+
+	p.ServeCodec(jsonrpc.NewServerCodec)
+	return nil
+}

pkg/plugin/config.go

@@ -0,0 +1,230 @@
+package plugin
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/stashapp/stash/pkg/models"
+	"gopkg.in/yaml.v2"
+)
+
+// Config describes the configuration for a single plugin.
+type Config struct {
+	id string
+
+	// path to the configuration file
+	path string
+
+	// The name of the plugin. This will be displayed in the UI.
+	Name string `yaml:"name"`
+
+	// An optional description of what the plugin does.
+	Description *string `yaml:"description"`
+
+	// An optional URL for the plugin.
+	URL *string `yaml:"url"`
+
+	// An optional version string.
+	Version *string `yaml:"version"`
+
+	// The communication interface used when communicating with the spawned
+	// plugin process. Defaults to 'raw' if not provided.
+	Interface interfaceEnum `yaml:"interface"`
+
+	// The command to execute for the operations in this plugin. The first
+	// element should be the program name, and subsequent elements are passed
+	// as arguments.
+	//
+	// Note: the execution process will search the path for the program,
+	// then will attempt to find the program in the plugins
+	// directory. The exe extension is not necessary on Windows platforms.
+	// The current working directory is set to that of the stash process.
+	Exec []string `yaml:"exec,flow"`
+
+	// The default log level to output the plugin process's stderr stream.
+	// Only used if the plugin does not encode its output using log level
+	// control characters.
+	// See package common/log for valid values.
+	// If left unset, defaults to log.ErrorLevel.
+	PluginErrLogLevel string `ymal:"errLog"`
+
+	// The task configurations for tasks provided by this plugin.
+	Tasks []*OperationConfig `yaml:"tasks"`
+}
+
+func (c Config) getPluginTasks(includePlugin bool) []*models.PluginTask {
+	var ret []*models.PluginTask
+
+	for _, o := range c.Tasks {
+		task := &models.PluginTask{
+			Name:        o.Name,
+			Description: &o.Description,
+		}
+
+		if includePlugin {
+			task.Plugin = c.toPlugin()
+		}
+		ret = append(ret, task)
+	}
+
+	return ret
+}
+
+func (c Config) getName() string {
+	if c.Name != "" {
+		return c.Name
+	}
+
+	return c.id
+}
+
+func (c Config) toPlugin() *models.Plugin {
+	return &models.Plugin{
+		ID:          c.id,
+		Name:        c.getName(),
+		Description: c.Description,
+		URL:         c.URL,
+		Version:     c.Version,
+		Tasks:       c.getPluginTasks(false),
+	}
+}
+
+func (c Config) getTask(name string) *OperationConfig {
+	for _, o := range c.Tasks {
+		if o.Name == name {
+			return o
+		}
+	}
+
+	return nil
+}
+
+func (c Config) getConfigPath() string {
+	return filepath.Dir(c.path)
+}
+
+func (c Config) getExecCommand(task *OperationConfig) []string {
+	ret := c.Exec
+
+	ret = append(ret, task.ExecArgs...)
+
+	if len(ret) > 0 {
+		_, err := exec.LookPath(ret[0])
+		if err != nil {
+			// change command to use absolute path
+			pluginPath := filepath.Dir(c.path)
+			ret[0] = filepath.Join(pluginPath, ret[0])
+		}
+	}
+
+	// replace {pluginDir} in arguments with that of the plugin directory
+	dir := c.getConfigPath()
+	for i, arg := range ret {
+		if i == 0 {
+			continue
+		}
+
+		ret[i] = strings.Replace(arg, "{pluginDir}", dir, -1)
+	}
+
+	return ret
+}
+
+type interfaceEnum string
+
+// Valid interfaceEnum values
+const (
+	// InterfaceEnumRPC indicates that the plugin uses the RPCRunner interface
+	// declared in common/rpc.go.
+	InterfaceEnumRPC interfaceEnum = "rpc"
+
+	// InterfaceEnumRaw interfaces will have the common.PluginInput encoded as
+	// json (but may be ignored), and output will be decoded as
+	// common.PluginOutput. If this decoding fails, then the raw output will be
+	// treated as the output.
+	InterfaceEnumRaw interfaceEnum = "raw"
+)
+
+func (i interfaceEnum) Valid() bool {
+	return i == InterfaceEnumRPC || i == InterfaceEnumRaw
+}
+
+func (i *interfaceEnum) getTaskBuilder() taskBuilder {
+	if *i == InterfaceEnumRaw {
+		return &rawTaskBuilder{}
+	}
+
+	if *i == InterfaceEnumRPC {
+		return &rpcTaskBuilder{}
+	}
+
+	// shouldn't happen
+	return nil
+}
+
+// OperationConfig describes the configuration for a single plugin operation
+// provided by a plugin.
+type OperationConfig struct {
+	// Used to identify the operation. Must be unique within a plugin
+	// configuration. This name is shown in the button for the operation
+	// in the UI.
+	Name string `yaml:"name"`
+
+	// A short description of the operation. This description is shown below
+	// the button in the UI.
+	Description string `yaml:"description"`
+
+	// A list of arguments that will be appended to the plugin's Exec arguments
+	// when executing this operation.
+	ExecArgs []string `yaml:"execArgs"`
+
+	// A map of argument keys to their default values. The default value is
+	// used if the applicable argument is not provided during the operation
+	// call.
+	DefaultArgs map[string]string `yaml:"defaultArgs"`
+}
+
+func loadPluginFromYAML(reader io.Reader) (*Config, error) {
+	ret := &Config{}
+
+	parser := yaml.NewDecoder(reader)
+	parser.SetStrict(true)
+	err := parser.Decode(&ret)
+	if err != nil {
+		return nil, err
+	}
+
+	if ret.Interface == "" {
+		ret.Interface = InterfaceEnumRaw
+	}
+
+	if !ret.Interface.Valid() {
+		return nil, fmt.Errorf("invalid interface type %s", ret.Interface)
+	}
+
+	return ret, nil
+}
+
+func loadPluginFromYAMLFile(path string) (*Config, error) {
+	file, err := os.Open(path)
+	defer file.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	ret, err := loadPluginFromYAML(file)
+	if err != nil {
+		return nil, err
+	}
+
+	// set id to the filename
+	id := filepath.Base(path)
+	ret.id = id[:strings.LastIndex(id, ".")]
+	ret.path = path
+
+	return ret, nil
+}

pkg/plugin/convert.go

@@ -0,0 +1,42 @@
+package plugin
+
+import (
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/plugin/common"
+)
+
+func toPluginArgs(args []*models.PluginArgInput) common.ArgsMap {
+	ret := make(common.ArgsMap)
+	for _, a := range args {
+		ret[a.Key] = toPluginArgValue(a.Value)
+	}
+
+	return ret
+}
+
+func toPluginArgValue(arg *models.PluginValueInput) common.PluginArgValue {
+	if arg == nil {
+		return nil
+	}
+
+	switch {
+	case arg.Str != nil:
+		return common.PluginArgValue(*arg.Str)
+	case arg.I != nil:
+		return common.PluginArgValue(*arg.I)
+	case arg.B != nil:
+		return common.PluginArgValue(*arg.B)
+	case arg.F != nil:
+		return common.PluginArgValue(*arg.F)
+	case arg.O != nil:
+		return common.PluginArgValue(toPluginArgs(arg.O))
+	case arg.A != nil:
+		var ret []common.PluginArgValue
+		for _, v := range arg.A {
+			ret = append(ret, toPluginArgValue(v))
+		}
+		return common.PluginArgValue(ret)
+	}
+
+	return nil
+}

pkg/plugin/examples/README.md

@@ -0,0 +1,9 @@
+# Building
+
+From the base stash source directory:
+```
+go build -tags=plugin_example -o plugin_goraw.exe ./pkg/plugin/examples/goraw/...
+go build -tags=plugin_example -o plugin_gorpc.exe ./pkg/plugin/examples/gorpc/...
+```
+
+Place the resulting binaries together with the yml files in the `plugins` subdirectory of your stash directory.

pkg/plugin/examples/common/graphql.go

@@ -0,0 +1,231 @@
+// +build plugin_example
+
+package common
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/shurcooL/graphql"
+	"github.com/stashapp/stash/pkg/plugin/common/log"
+)
+
+const tagName = "Hawwwwt"
+
+// graphql inputs and returns
+type TagCreate struct {
+	ID graphql.ID `graphql:"id"`
+}
+
+type TagCreateInput struct {
+	Name graphql.String `graphql:"name" json:"name"`
+}
+
+type TagDestroyInput struct {
+	ID graphql.ID `graphql:"id" json:"id"`
+}
+
+type FindScenesResultType struct {
+	Count  graphql.Int
+	Scenes []Scene
+}
+
+type Tag struct {
+	ID   graphql.ID     `graphql:"id"`
+	Name graphql.String `graphql:"name"`
+}
+
+type Scene struct {
+	ID   graphql.ID
+	Tags []Tag
+}
+
+func (s Scene) getTagIds() []graphql.ID {
+	ret := []graphql.ID{}
+
+	for _, t := range s.Tags {
+		ret = append(ret, t.ID)
+	}
+
+	return ret
+}
+
+type FindFilterType struct {
+	PerPage *graphql.Int    `graphql:"per_page" json:"per_page"`
+	Sort    *graphql.String `graphql:"sort" json:"sort"`
+}
+
+type SceneUpdate struct {
+	ID graphql.ID `graphql:"id"`
+}
+
+type SceneUpdateInput struct {
+	ID     graphql.ID   `graphql:"id" json:"id"`
+	TagIds []graphql.ID `graphql:"tag_ids" json:"tag_ids"`
+}
+
+func getTagID(client *graphql.Client, create bool) (*graphql.ID, error) {
+	log.Info("Checking if tag exists already")
+
+	// see if tag exists already
+	var q struct {
+		AllTags []Tag `graphql:"allTags"`
+	}
+
+	err := client.Query(context.Background(), &q, nil)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting tags: %s\n", err.Error())
+	}
+
+	for _, t := range q.AllTags {
+		if t.Name == tagName {
+			id := t.ID
+			return &id, nil
+		}
+	}
+
+	if !create {
+		log.Info("Not found and not creating")
+		return nil, nil
+	}
+
+	// create the tag
+	var m struct {
+		TagCreate TagCreate `graphql:"tagCreate(input: $s)"`
+	}
+
+	input := TagCreateInput{
+		Name: tagName,
+	}
+
+	vars := map[string]interface{}{
+		"s": input,
+	}
+
+	log.Info("Creating new tag")
+
+	err = client.Mutate(context.Background(), &m, vars)
+	if err != nil {
+		return nil, fmt.Errorf("Error mutating scene: %s\n", err.Error())
+	}
+
+	return &m.TagCreate.ID, nil
+}
+
+func findRandomScene(client *graphql.Client) (*Scene, error) {
+	// get a random scene
+	var q struct {
+		FindScenes FindScenesResultType `graphql:"findScenes(filter: $c)"`
+	}
+
+	pp := graphql.Int(1)
+	sort := graphql.String("random")
+	filterInput := &FindFilterType{
+		PerPage: &pp,
+		Sort:    &sort,
+	}
+
+	vars := map[string]interface{}{
+		"c": filterInput,
+	}
+
+	log.Info("Finding a random scene")
+	err := client.Query(context.Background(), &q, vars)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting random scene: %s\n", err.Error())
+	}
+
+	if q.FindScenes.Count == 0 {
+		return nil, nil
+	}
+
+	return &q.FindScenes.Scenes[0], nil
+}
+
+func addTagId(tagIds []graphql.ID, tagId graphql.ID) []graphql.ID {
+	for _, t := range tagIds {
+		if t == tagId {
+			return tagIds
+		}
+	}
+
+	tagIds = append(tagIds, tagId)
+	return tagIds
+}
+
+func AddTag(client *graphql.Client) error {
+	tagID, err := getTagID(client, true)
+
+	if err != nil {
+		return err
+	}
+
+	scene, err := findRandomScene(client)
+
+	if err != nil {
+		return err
+	}
+
+	if scene == nil {
+		return errors.New("no scenes to add tag to")
+	}
+
+	var m struct {
+		SceneUpdate SceneUpdate `graphql:"sceneUpdate(input: $s)"`
+	}
+
+	input := SceneUpdateInput{
+		ID:     scene.ID,
+		TagIds: scene.getTagIds(),
+	}
+
+	input.TagIds = addTagId(input.TagIds, *tagID)
+
+	vars := map[string]interface{}{
+		"s": input,
+	}
+
+	log.Infof("Adding tag to scene %v", scene.ID)
+	err = client.Mutate(context.Background(), &m, vars)
+	if err != nil {
+		return fmt.Errorf("Error mutating scene: %s", err.Error())
+	}
+
+	return nil
+}
+
+func RemoveTag(client *graphql.Client) error {
+	tagID, err := getTagID(client, false)
+
+	if err != nil {
+		return err
+	}
+
+	if tagID == nil {
+		log.Info("Tag does not exist. Nothing to remove")
+		return nil
+	}
+
+	// destroy the tag
+	var m struct {
+		TagDestroy bool `graphql:"tagDestroy(input: $s)"`
+	}
+
+	input := TagDestroyInput{
+		ID: *tagID,
+	}
+
+	vars := map[string]interface{}{
+		"s": input,
+	}
+
+	log.Info("Destroying tag")
+
+	err = client.Mutate(context.Background(), &m, vars)
+	if err != nil {
+		return fmt.Errorf("Error destroying tag: %s", err.Error())
+	}
+
+	return nil
+}

pkg/plugin/examples/goraw/goraw.yml

@@ -0,0 +1,28 @@
+# example plugin config
+name: Hawwwwt Tagger (Raw edition)
+description: Ultimate Hawwwwt tagging utility (using raw interface).
+version: 1.0
+url: http://www.github.com/stashapp/stash
+exec:
+  - plugin_goraw
+interface: raw
+tasks:
+  - name: Add hawwwwt tag to random scene
+    description: Creates a "Hawwwwt" tag if not present and adds to a random scene.
+    defaultArgs:
+      mode: add
+  - name: Remove hawwwwt tag from system
+    description: Removes the "Hawwwwt" tag from all scenes and deletes the tag.
+    defaultArgs:
+      mode: remove
+  - name: Indefinite task
+    description: Sleeps indefinitely - interruptable
+    # we'll try command-line argument for this one
+    execArgs:
+      - indef
+      - "{pluginDir}"
+  - name: Long task
+    description: Sleeps for 100 seconds - interruptable
+    defaultArgs:
+      mode: long
+      

pkg/plugin/examples/goraw/main.go

@@ -0,0 +1,106 @@
+// +build plugin_example
+
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"time"
+
+	exampleCommon "github.com/stashapp/stash/pkg/plugin/examples/common"
+
+	"github.com/stashapp/stash/pkg/plugin/common"
+	"github.com/stashapp/stash/pkg/plugin/common/log"
+	"github.com/stashapp/stash/pkg/plugin/util"
+)
+
+// raw plugins may accept the plugin input from stdin, or they can elect
+// to ignore it entirely. In this case it optionally reads from the
+// command-line parameters.
+func main() {
+	input := common.PluginInput{}
+
+	if len(os.Args) < 2 {
+		inData, _ := ioutil.ReadAll(os.Stdin)
+		log.Debugf("Raw input: %s", string(inData))
+		decodeErr := json.Unmarshal(inData, &input)
+
+		if decodeErr != nil {
+			panic("missing mode argument")
+		}
+	} else {
+		log.Debug("Using command line inputs")
+		mode := os.Args[1]
+		log.Debugf("Command line inputs: %v", os.Args[1:])
+		input.Args = common.ArgsMap{
+			"mode": mode,
+		}
+
+		// just some hard-coded values
+		input.ServerConnection = common.StashServerConnection{
+			Scheme: "http",
+			Port:   9999,
+		}
+	}
+
+	output := common.PluginOutput{}
+	Run(input, &output)
+
+	out, _ := json.Marshal(output)
+	os.Stdout.WriteString(string(out))
+}
+
+func Run(input common.PluginInput, output *common.PluginOutput) error {
+	modeArg := input.Args.String("mode")
+
+	var err error
+	if modeArg == "" || modeArg == "add" {
+		client := util.NewClient(input.ServerConnection)
+		err = exampleCommon.AddTag(client)
+	} else if modeArg == "remove" {
+		client := util.NewClient(input.ServerConnection)
+		err = exampleCommon.RemoveTag(client)
+	} else if modeArg == "long" {
+		err = doLongTask()
+	} else if modeArg == "indef" {
+		err = doIndefiniteTask()
+	}
+
+	if err != nil {
+		errStr := err.Error()
+		*output = common.PluginOutput{
+			Error: &errStr,
+		}
+		return nil
+	}
+
+	outputStr := "ok"
+	*output = common.PluginOutput{
+		Output: &outputStr,
+	}
+
+	return nil
+}
+
+func doLongTask() error {
+	const total = 100
+	upTo := 0
+
+	log.Info("Doing long task")
+	for upTo < total {
+		time.Sleep(time.Second)
+
+		log.Progress(float64(upTo) / float64(total))
+		upTo++
+	}
+
+	return nil
+}
+
+func doIndefiniteTask() error {
+	log.Warn("Sleeping indefinitely")
+	for {
+		time.Sleep(time.Second)
+	}
+}

pkg/plugin/examples/gorpc/gorpc.yml

@@ -0,0 +1,26 @@
+# example plugin config
+name: Hawwwwt Tagger
+description: Ultimate Hawwwwt tagging utility.
+version: 1.0
+url: http://www.github.com/stashapp/stash
+exec:
+  - plugin_gorpc
+interface: rpc
+tasks:
+  - name: Add hawwwwt tag to random scene
+    description: Creates a "Hawwwwt" tag if not present and adds to a random scene.
+    defaultArgs:
+      mode: add
+  - name: Remove hawwwwt tag from system
+    description: Removes the "Hawwwwt" tag from all scenes and deletes the tag.
+    defaultArgs:
+      mode: remove
+  - name: Indefinite task
+    description: Sleeps indefinitely - interruptable
+    defaultArgs:
+      mode: indef
+  - name: Long task
+    description: Sleeps for 100 seconds - interruptable
+    defaultArgs:
+      mode: long
+      

pkg/plugin/examples/gorpc/main.go

@@ -0,0 +1,97 @@
+// +build plugin_example
+
+package main
+
+import (
+	"time"
+
+	exampleCommon "github.com/stashapp/stash/pkg/plugin/examples/common"
+
+	"github.com/stashapp/stash/pkg/plugin/common"
+	"github.com/stashapp/stash/pkg/plugin/common/log"
+	"github.com/stashapp/stash/pkg/plugin/util"
+)
+
+func main() {
+	// serves the plugin, providing an object that satisfies the
+	// common.RPCRunner interface
+	err := common.ServePlugin(&api{})
+	if err != nil {
+		panic(err)
+	}
+}
+
+type api struct {
+	stopping bool
+}
+
+func (a *api) Stop(input struct{}, output *bool) error {
+	log.Info("Stopping...")
+	a.stopping = true
+	*output = true
+	return nil
+}
+
+// Run is the main work function of the plugin. It interprets the input and
+// acts accordingly.
+func (a *api) Run(input common.PluginInput, output *common.PluginOutput) error {
+	modeArg := input.Args.String("mode")
+
+	var err error
+	if modeArg == "" || modeArg == "add" {
+		client := util.NewClient(input.ServerConnection)
+		err = exampleCommon.AddTag(client)
+	} else if modeArg == "remove" {
+		client := util.NewClient(input.ServerConnection)
+		err = exampleCommon.RemoveTag(client)
+	} else if modeArg == "long" {
+		err = a.doLongTask()
+	} else if modeArg == "indef" {
+		err = a.doIndefiniteTask()
+	}
+
+	if err != nil {
+		errStr := err.Error()
+		*output = common.PluginOutput{
+			Error: &errStr,
+		}
+		return nil
+	}
+
+	outputStr := "ok"
+	*output = common.PluginOutput{
+		Output: &outputStr,
+	}
+
+	return nil
+}
+
+func (a *api) doLongTask() error {
+	const total = 100
+	upTo := 0
+
+	log.Info("Doing long task")
+	for upTo < total {
+		time.Sleep(time.Second)
+		if a.stopping {
+			return nil
+		}
+
+		log.Progress(float64(upTo) / float64(total))
+		upTo++
+	}
+
+	return nil
+}
+
+func (a *api) doIndefiniteTask() error {
+	log.Warn("Sleeping indefinitely")
+	for {
+		time.Sleep(time.Second)
+		if a.stopping {
+			return nil
+		}
+	}
+
+	return nil
+}

pkg/plugin/log.go

@@ -0,0 +1,79 @@
+package plugin
+
+import (
+	"bufio"
+	"io"
+	"strconv"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/plugin/common/log"
+)
+
+func (t *pluginTask) handleStderrLine(line string, defaultLogLevel *log.Level) {
+	level, l := log.DetectLogLevel(line)
+
+	const pluginPrefix = "[Plugin] "
+	// if no log level, just output to info
+	if level == nil {
+		if defaultLogLevel != nil {
+			level = defaultLogLevel
+		} else {
+			level = &log.InfoLevel
+		}
+	}
+
+	switch *level {
+	case log.TraceLevel:
+		logger.Trace(pluginPrefix, l)
+	case log.DebugLevel:
+		logger.Debug(pluginPrefix, l)
+	case log.InfoLevel:
+		logger.Info(pluginPrefix, l)
+	case log.WarningLevel:
+		logger.Warn(pluginPrefix, l)
+	case log.ErrorLevel:
+		logger.Error(pluginPrefix, l)
+	case log.ProgressLevel:
+		progress, err := strconv.ParseFloat(l, 64)
+		if err != nil {
+			logger.Errorf("Error parsing progress value '%s': %s", l, err.Error())
+		} else {
+			// only pass progress through if channel present
+			if t.progress != nil {
+				// don't block on this
+				select {
+				case t.progress <- progress:
+				default:
+				}
+			}
+		}
+	}
+}
+
+func (t *pluginTask) handlePluginOutput(pluginOutputReader io.ReadCloser, defaultLogLevel *log.Level) {
+	// pipe plugin stderr to our logging
+	scanner := bufio.NewScanner(pluginOutputReader)
+	for scanner.Scan() {
+		str := scanner.Text()
+		if str != "" {
+			t.handleStderrLine(str, defaultLogLevel)
+		}
+	}
+
+	str := scanner.Text()
+	if str != "" {
+		t.handleStderrLine(str, defaultLogLevel)
+	}
+
+	pluginOutputReader.Close()
+}
+
+func (t *pluginTask) handlePluginStderr(pluginOutputReader io.ReadCloser) {
+	logLevel := log.LevelFromName(t.plugin.PluginErrLogLevel)
+	if logLevel == nil {
+		// default log level to error
+		logLevel = &log.ErrorLevel
+	}
+
+	t.handlePluginOutput(pluginOutputReader, logLevel)
+}

pkg/plugin/plugins.go

@@ -0,0 +1,140 @@
+// Package plugin implements functions and types for maintaining and running
+// stash plugins.
+//
+// Stash plugins are configured using yml files in the configured plugins
+// directory. These yml files must follow the Config structure format.
+//
+// The main entry into the plugin sub-system is via the Cache type.
+package plugin
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/plugin/common"
+)
+
+// Cache stores plugin details.
+type Cache struct {
+	path    string
+	plugins []Config
+}
+
+// NewCache returns a new Cache loading plugin configurations
+// from the provided plugin path. It returns an new instance and an error
+// if the plugin directory could not be loaded.
+//
+// Plugins configurations are loaded from yml files in the provided plugin
+// directory and any subdirectories.
+func NewCache(pluginPath string) (*Cache, error) {
+	plugins, err := loadPlugins(pluginPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Cache{
+		path:    pluginPath,
+		plugins: plugins,
+	}, nil
+}
+
+// ReloadPlugins clears the plugin cache and reloads from the plugin path.
+// In the event of an error during loading, the cache will be left empty.
+func (c *Cache) ReloadPlugins() error {
+	c.plugins = nil
+	plugins, err := loadPlugins(c.path)
+	if err != nil {
+		return err
+	}
+
+	c.plugins = plugins
+	return nil
+}
+
+func loadPlugins(path string) ([]Config, error) {
+	plugins := make([]Config, 0)
+
+	logger.Debugf("Reading plugin configs from %s", path)
+	pluginFiles := []string{}
+	err := filepath.Walk(path, func(fp string, f os.FileInfo, err error) error {
+		if filepath.Ext(fp) == ".yml" {
+			pluginFiles = append(pluginFiles, fp)
+		}
+		return nil
+	})
+
+	if err != nil {
+
+		return nil, err
+	}
+
+	for _, file := range pluginFiles {
+		plugin, err := loadPluginFromYAMLFile(file)
+		if err != nil {
+			logger.Errorf("Error loading plugin %s: %s", file, err.Error())
+		} else {
+			plugins = append(plugins, *plugin)
+		}
+	}
+
+	return plugins, nil
+}
+
+// ListPlugins returns plugin details for all of the loaded plugins.
+func (c Cache) ListPlugins() []*models.Plugin {
+	var ret []*models.Plugin
+	for _, s := range c.plugins {
+		ret = append(ret, s.toPlugin())
+	}
+
+	return ret
+}
+
+// ListPluginTasks returns all runnable plugin tasks in all loaded plugins.
+func (c Cache) ListPluginTasks() []*models.PluginTask {
+	var ret []*models.PluginTask
+	for _, s := range c.plugins {
+		ret = append(ret, s.getPluginTasks(true)...)
+	}
+
+	return ret
+}
+
+// CreateTask runs the plugin operation for the pluginID and operation
+// name provided. Returns an error if the plugin or the operation could not be
+// resolved.
+func (c Cache) CreateTask(pluginID string, operationName string, serverConnection common.StashServerConnection, args []*models.PluginArgInput, progress chan float64) (Task, error) {
+	// find the plugin and operation
+	plugin := c.getPlugin(pluginID)
+
+	if plugin == nil {
+		return nil, fmt.Errorf("no plugin with ID %s", pluginID)
+	}
+
+	operation := plugin.getTask(operationName)
+	if operation == nil {
+		return nil, fmt.Errorf("no task with name %s in plugin %s", operationName, plugin.getName())
+	}
+
+	task := pluginTask{
+		plugin:           plugin,
+		operation:        operation,
+		serverConnection: serverConnection,
+		args:             args,
+		progress:         progress,
+	}
+	return task.createTask(), nil
+}
+
+func (c Cache) getPlugin(pluginID string) *Config {
+	for _, s := range c.plugins {
+		if s.id == pluginID {
+			return &s
+		}
+	}
+
+	return nil
+}

pkg/plugin/raw.go

@@ -0,0 +1,122 @@
+package plugin
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os/exec"
+	"sync"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/plugin/common"
+)
+
+type rawTaskBuilder struct{}
+
+func (*rawTaskBuilder) build(task pluginTask) Task {
+	return &rawPluginTask{
+		pluginTask: task,
+	}
+}
+
+type rawPluginTask struct {
+	pluginTask
+
+	started   bool
+	waitGroup sync.WaitGroup
+	cmd       *exec.Cmd
+	done      chan bool
+}
+
+func (t *rawPluginTask) Start() error {
+	if t.started {
+		return errors.New("task already started")
+	}
+
+	command := t.plugin.getExecCommand(t.operation)
+	if len(command) == 0 {
+		return fmt.Errorf("empty exec value in operation %s", t.operation.Name)
+	}
+
+	cmd := exec.Command(command[0], command[1:]...)
+
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("error getting plugin process stdin: %s", err.Error())
+	}
+
+	go func() {
+		defer stdin.Close()
+
+		input := t.buildPluginInput()
+		inBytes, _ := json.Marshal(input)
+		io.WriteString(stdin, string(inBytes))
+	}()
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		logger.Error("Plugin stderr not available: " + err.Error())
+	}
+
+	stdout, err := cmd.StdoutPipe()
+	if nil != err {
+		logger.Error("Plugin stdout not available: " + err.Error())
+	}
+
+	t.waitGroup.Add(1)
+	t.done = make(chan bool, 1)
+	if err = cmd.Start(); err != nil {
+		return fmt.Errorf("Error running plugin: %s", err.Error())
+	}
+
+	go t.handlePluginStderr(stderr)
+	t.cmd = cmd
+
+	// send the stdout to the plugin output
+	go func() {
+		defer t.waitGroup.Done()
+		defer close(t.done)
+		stdoutData, _ := ioutil.ReadAll(stdout)
+		stdoutString := string(stdoutData)
+
+		output := t.getOutput(stdoutString)
+
+		err := cmd.Wait()
+		if err != nil && output.Error == nil {
+			errStr := err.Error()
+			output.Error = &errStr
+		}
+
+		t.result = &output
+	}()
+
+	t.started = true
+	return nil
+}
+
+func (t *rawPluginTask) getOutput(output string) common.PluginOutput {
+	// try to parse the output as a PluginOutput json. If it fails just
+	// get the raw output
+	ret := common.PluginOutput{}
+	decodeErr := json.Unmarshal([]byte(output), &ret)
+
+	if decodeErr != nil {
+		ret.Output = &output
+	}
+
+	return ret
+}
+
+func (t *rawPluginTask) Wait() {
+	t.waitGroup.Wait()
+}
+
+func (t *rawPluginTask) Stop() error {
+	if t.cmd == nil {
+		return nil
+	}
+
+	return t.cmd.Process.Kill()
+}

pkg/plugin/rpc.go

@@ -0,0 +1,103 @@
+package plugin
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/rpc"
+	"net/rpc/jsonrpc"
+	"sync"
+
+	"github.com/natefinch/pie"
+	"github.com/stashapp/stash/pkg/plugin/common"
+)
+
+type rpcTaskBuilder struct{}
+
+func (*rpcTaskBuilder) build(task pluginTask) Task {
+	return &rpcPluginTask{
+		pluginTask: task,
+	}
+}
+
+type rpcPluginClient struct {
+	Client *rpc.Client
+}
+
+func (p rpcPluginClient) Run(input common.PluginInput, output *common.PluginOutput) error {
+	return p.Client.Call("RPCRunner.Run", input, output)
+}
+
+func (p rpcPluginClient) RunAsync(input common.PluginInput, output *common.PluginOutput, done chan *rpc.Call) *rpc.Call {
+	return p.Client.Go("RPCRunner.Run", input, output, done)
+}
+
+func (p rpcPluginClient) Stop() error {
+	var resp interface{}
+	return p.Client.Call("RPCRunner.Stop", nil, &resp)
+}
+
+type rpcPluginTask struct {
+	pluginTask
+
+	started   bool
+	client    *rpc.Client
+	waitGroup sync.WaitGroup
+	done      chan *rpc.Call
+}
+
+func (t *rpcPluginTask) Start() error {
+	if t.started {
+		return errors.New("task already started")
+	}
+
+	command := t.plugin.getExecCommand(t.operation)
+	if len(command) == 0 {
+		return fmt.Errorf("empty exec value in operation %s", t.operation.Name)
+	}
+
+	pluginErrReader, pluginErrWriter := io.Pipe()
+
+	var err error
+	t.client, err = pie.StartProviderCodec(jsonrpc.NewClientCodec, pluginErrWriter, command[0], command[1:]...)
+	if err != nil {
+		return err
+	}
+
+	go t.handlePluginStderr(pluginErrReader)
+
+	iface := rpcPluginClient{
+		Client: t.client,
+	}
+
+	input := t.buildPluginInput()
+
+	t.done = make(chan *rpc.Call, 1)
+	result := common.PluginOutput{}
+	t.waitGroup.Add(1)
+	iface.RunAsync(input, &result, t.done)
+	go t.waitToFinish(&result)
+
+	t.started = true
+	return nil
+}
+
+func (t *rpcPluginTask) waitToFinish(result *common.PluginOutput) {
+	defer t.client.Close()
+	defer t.waitGroup.Done()
+	<-t.done
+
+	t.result = result
+}
+
+func (t *rpcPluginTask) Wait() {
+	t.waitGroup.Wait()
+}
+
+func (t *rpcPluginTask) Stop() error {
+	iface := rpcPluginClient{
+		Client: t.client,
+	}
+
+	return iface.Stop()
+}

pkg/plugin/task.go

@@ -0,0 +1,56 @@
+package plugin
+
+import (
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/plugin/common"
+)
+
+// Task is the interface that handles management of a single plugin task.
+type Task interface {
+	// Start starts the plugin task. Returns an error if task could not be
+	// started or the task has already been started.
+	Start() error
+
+	// Stop instructs a running plugin task to stop and returns immediately.
+	// Use Wait to subsequently wait for the task to stop.
+	Stop() error
+
+	// Wait blocks until the plugin task is complete. Returns immediately if
+	// task has not been started.
+	Wait()
+
+	// GetResult returns the output of the plugin task. Returns nil if the task
+	// has not completed.
+	GetResult() *common.PluginOutput
+}
+
+type taskBuilder interface {
+	build(task pluginTask) Task
+}
+
+type pluginTask struct {
+	plugin           *Config
+	operation        *OperationConfig
+	serverConnection common.StashServerConnection
+	args             []*models.PluginArgInput
+
+	progress chan float64
+	result   *common.PluginOutput
+}
+
+func (t *pluginTask) GetResult() *common.PluginOutput {
+	return t.result
+}
+
+func (t *pluginTask) createTask() Task {
+	return t.plugin.Interface.getTaskBuilder().build(*t)
+}
+
+func (t *pluginTask) buildPluginInput() common.PluginInput {
+	args := applyDefaultArgs(t.args, t.operation.DefaultArgs)
+	t.serverConnection.PluginDir = t.plugin.getConfigPath()
+	return common.PluginInput{
+		ServerConnection: t.serverConnection,
+		Args:             toPluginArgs(args),
+	}
+}

pkg/plugin/util/client.go

@@ -0,0 +1,39 @@
+// Package util implements utility and convenience methods for plugins. It is
+// not intended for the main stash code to access.
+package util
+
+import (
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strconv"
+
+	"github.com/shurcooL/graphql"
+
+	"github.com/stashapp/stash/pkg/plugin/common"
+)
+
+// NewClient creates a graphql Client connecting to the stash server using
+// the provided server connection details.
+// Always connects to the graphql endpoint of the localhost.
+func NewClient(provider common.StashServerConnection) *graphql.Client {
+	portStr := strconv.Itoa(provider.Port)
+
+	u, _ := url.Parse("http://localhost:" + portStr + "/graphql")
+	u.Scheme = provider.Scheme
+
+	cookieJar, _ := cookiejar.New(nil)
+
+	cookie := provider.SessionCookie
+	if cookie != nil {
+		cookieJar.SetCookies(u, []*http.Cookie{
+			cookie,
+		})
+	}
+
+	httpClient := &http.Client{
+		Jar: cookieJar,
+	}
+
+	return graphql.NewClient(u.String(), httpClient)
+}

ui/v2.5/src/components/Changelog/versions/v030.tsx

@@ -5,6 +5,7 @@ const markup = `
 #### 💥 **Note: After upgrading, the next scan will populate all scenes with oshash hashes. MD5 calculation can be disabled after populating the oshash for all scenes. See \`Hashing Algorithms\` in the \`Configuration\` section of the manual for details. **
 
 ### ✨ New Features
+*  Add support for plugin tasks.
 *  Add oshash algorithm for hashing scene video files. Enabled by default on new systems.
 *  Support (re-)generation of generated content for specific scenes.
 *  Add tag thumbnails, tags grid view and tag page.

ui/v2.5/src/components/Help/Manual.tsx

@@ -8,6 +8,7 @@ import Configuration from "src/docs/en/Configuration.md";
 import Interface from "src/docs/en/Interface.md";
 import Galleries from "src/docs/en/Galleries.md";
 import Scraping from "src/docs/en/Scraping.md";
+import Plugins from "src/docs/en/Plugins.md";
 import Contributing from "src/docs/en/Contributing.md";
 import SceneFilenameParser from "src/docs/en/SceneFilenameParser.md";
 import KeyboardShortcuts from "src/docs/en/KeyboardShortcuts.md";
@@ -69,6 +70,11 @@ export const Manual: React.FC<IManualProps> = ({ show, onClose }) => {
       title: "Metadata Scraping",
       content: Scraping,
     },
+    {
+      key: "Plugins.md",
+      title: "Plugins",
+      content: Plugins,
+    },
     {
       key: "KeyboardShortcuts.md",
       title: "Keyboard Shortcuts",

ui/v2.5/src/components/Settings/Settings.tsx

@@ -7,6 +7,7 @@ import { SettingsConfigurationPanel } from "./SettingsConfigurationPanel";
 import { SettingsInterfacePanel } from "./SettingsInterfacePanel";
 import { SettingsLogsPanel } from "./SettingsLogsPanel";
 import { SettingsTasksPanel } from "./SettingsTasksPanel/SettingsTasksPanel";
+import { SettingsPluginsPanel } from "./SettingsPluginsPanel";
 
 export const Settings: React.FC = () => {
   const location = useLocation();
@@ -34,6 +35,9 @@ export const Settings: React.FC = () => {
               <Nav.Item>
                 <Nav.Link eventKey="tasks">Tasks</Nav.Link>
               </Nav.Item>
+              <Nav.Item>
+                <Nav.Link eventKey="plugins">Plugins</Nav.Link>
+              </Nav.Item>
               <Nav.Item>
                 <Nav.Link eventKey="logs">Logs</Nav.Link>
               </Nav.Item>
@@ -54,6 +58,9 @@ export const Settings: React.FC = () => {
               <Tab.Pane eventKey="tasks">
                 <SettingsTasksPanel />
               </Tab.Pane>
+              <Tab.Pane eventKey="plugins">
+                <SettingsPluginsPanel />
+              </Tab.Pane>
               <Tab.Pane eventKey="logs">
                 <SettingsLogsPanel />
               </Tab.Pane>

ui/v2.5/src/components/Settings/SettingsPluginsPanel.tsx

@@ -0,0 +1,92 @@
+import React, { useState, useEffect } from "react";
+import { Button } from "react-bootstrap";
+import { mutateReloadPlugins, usePlugins } from "src/core/StashService";
+import { useToast } from "src/hooks";
+import * as GQL from "src/core/generated-graphql";
+import { TextUtils } from "src/utils";
+import { Icon, LoadingIndicator } from "../Shared";
+
+export const SettingsPluginsPanel: React.FC = () => {
+  const Toast = useToast();
+
+  const plugins = usePlugins();
+
+  // Network state
+  const [isLoading, setIsLoading] = useState(true);
+
+  useEffect(() => {
+    if (plugins) {
+      setIsLoading(false);
+    }
+  }, [plugins]);
+
+  async function onReloadPlugins() {
+    setIsLoading(true);
+    try {
+      await mutateReloadPlugins();
+
+      // reload the performer scrapers
+      await plugins.refetch();
+    } catch (e) {
+      Toast.error(e);
+    } finally {
+      setIsLoading(false);
+    }
+  }
+
+  function renderLink(plugin: GQL.Plugin) {
+    if (plugin.url) {
+      return (
+        <Button className="minimal">
+          <a
+            href={TextUtils.sanitiseURL(plugin.url)}
+            className="link"
+            target="_blank"
+            rel="noopener noreferrer"
+          >
+            <Icon icon="link" />
+          </a>
+        </Button>
+      );
+    }
+  }
+
+  function renderPlugin(plugin: GQL.Plugin) {
+    return (
+      <div key={plugin.id}>
+        <h5>
+          {plugin.name} {plugin.version ? `(${plugin.version})` : undefined}{" "}
+          {renderLink(plugin)}
+        </h5>
+        {plugin.description ? (
+          <small className="text-muted">{plugin.description}</small>
+        ) : undefined}
+        <hr />
+      </div>
+    );
+  }
+
+  function renderPlugins() {
+    if (!plugins.data || !plugins.data.plugins) {
+      return;
+    }
+
+    return <div>{plugins.data?.plugins.map(renderPlugin)}</div>;
+  }
+
+  if (isLoading) return <LoadingIndicator />;
+
+  return (
+    <>
+      <h4>Plugins</h4>
+      <hr />
+      {renderPlugins()}
+      <Button onClick={() => onReloadPlugins()}>
+        <span className="fa-icon">
+          <Icon icon="sync-alt" />
+        </span>
+        <span>Reload plugins</span>
+      </Button>
+    </>
+  );
+};

ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx

@@ -11,9 +11,12 @@ import {
   mutateMetadataExport,
   mutateMigrateHashNaming,
   mutateStopJob,
+  usePlugins,
+  mutateRunPluginTask,
 } from "src/core/StashService";
 import { useToast } from "src/hooks";
 import { Modal } from "src/components/Shared";
+import { Plugin, PluginTask } from "src/core/generated-graphql";
 import { GenerateButton } from "./GenerateButton";
 
 export const SettingsTasksPanel: React.FC = () => {
@@ -31,6 +34,8 @@ export const SettingsTasksPanel: React.FC = () => {
   const jobStatus = useJobStatus();
   const metadataUpdate = useMetadataUpdate();
 
+  const plugins = usePlugins();
+
   function statusToText(s: string) {
     switch (s) {
       case "Idle":
@@ -47,6 +52,8 @@ export const SettingsTasksPanel: React.FC = () => {
         return "Importing from JSON";
       case "Auto Tag":
         return "Auto tagging scenes";
+      case "Plugin Operation":
+        return "Running Plugin Operation";
       case "Migrate":
         return "Migrating";
       default:
@@ -59,7 +66,7 @@ export const SettingsTasksPanel: React.FC = () => {
       setStatus(statusToText(jobStatus.data.jobStatus.status));
       const newProgress = jobStatus.data.jobStatus.progress;
       if (newProgress < 0) {
-        setProgress(0);
+        setProgress(-1);
       } else {
         setProgress(newProgress * 100);
       }
@@ -71,7 +78,7 @@ export const SettingsTasksPanel: React.FC = () => {
       setStatus(statusToText(metadataUpdate.data.metadataUpdate.status));
       const newProgress = metadataUpdate.data.metadataUpdate.progress;
       if (newProgress < 0) {
-        setProgress(0);
+        setProgress(-1);
       } else {
         setProgress(newProgress * 100);
       }
@@ -180,8 +187,8 @@ export const SettingsTasksPanel: React.FC = () => {
           {!!status && status !== "Idle" ? (
             <ProgressBar
               animated
-              now={progress}
-              label={`${progress.toFixed(0)}%`}
+              now={progress > -1 ? progress : 100}
+              label={progress > -1 ? `${progress.toFixed(0)}%` : ""}
             />
           ) : (
             ""
@@ -192,6 +199,62 @@ export const SettingsTasksPanel: React.FC = () => {
     );
   }
 
+  async function onPluginTaskClicked(
+    plugin: Partial<Plugin>,
+    operation: Partial<PluginTask>
+  ) {
+    await mutateRunPluginTask(plugin.id!, operation.name!);
+  }
+
+  function renderPluginTasks(
+    plugin: Partial<Plugin>,
+    pluginTasks: Partial<PluginTask>[] | undefined
+  ) {
+    if (!pluginTasks) {
+      return;
+    }
+
+    return pluginTasks.map((o) => {
+      return (
+        <div key={o.name}>
+          <Button
+            onClick={() => onPluginTaskClicked(plugin, o)}
+            className="mt-3"
+            variant="secondary"
+            size="sm"
+          >
+            {o.name}
+          </Button>
+          {o.description ? (
+            <Form.Text className="text-muted">{o.description}</Form.Text>
+          ) : undefined}
+        </div>
+      );
+    });
+  }
+
+  function renderPlugins() {
+    if (!plugins.data || !plugins.data.plugins) {
+      return;
+    }
+
+    return (
+      <>
+        <hr />
+        <h5>Plugin Tasks</h5>
+        {plugins.data.plugins.map((o) => {
+          return (
+            <div key={`${o.id}`} className="mb-3">
+              <h6>{o.name}</h6>
+              {renderPluginTasks(o, o.tasks ?? [])}
+              <hr />
+            </div>
+          );
+        })}
+      </>
+    );
+  }
+
   return (
     <>
       {renderImportAlert()}
@@ -312,6 +375,8 @@ export const SettingsTasksPanel: React.FC = () => {
         </Form.Text>
       </Form.Group>
 
+      {renderPlugins()}
+
       <hr />
 
       <h5>Migrations</h5>

ui/v2.5/src/core/StashService.ts

@@ -170,6 +170,10 @@ export const useListSceneScrapers = () => GQL.useListSceneScrapersQuery();
 
 export const useScrapeFreeonesPerformers = (q: string) =>
   GQL.useScrapeFreeonesPerformersQuery({ variables: { q } });
+
+export const usePlugins = () => GQL.usePluginsQuery();
+export const usePluginTasks = () => GQL.usePluginTasksQuery();
+
 export const useMarkerStrings = () => GQL.useMarkerStringsQuery();
 export const useAllTags = () => GQL.useAllTagsQuery();
 export const useAllTagsForFilter = () => GQL.useAllTagsForFilterQuery();
@@ -446,6 +450,24 @@ export const mutateReloadScrapers = () =>
     mutation: GQL.ReloadScrapersDocument,
   });
 
+const reloadPluginsMutationImpactedQueries = ["plugins", "pluginTasks"];
+
+export const mutateReloadPlugins = () =>
+  client.mutate<GQL.ReloadPluginsMutation>({
+    mutation: GQL.ReloadPluginsDocument,
+    update: () => invalidateQueries(reloadPluginsMutationImpactedQueries),
+  });
+
+export const mutateRunPluginTask = (
+  pluginId: string,
+  taskName: string,
+  args?: GQL.PluginArgInput[]
+) =>
+  client.mutate<GQL.RunPluginTaskMutation>({
+    mutation: GQL.RunPluginTaskDocument,
+    variables: { plugin_id: pluginId, task_name: taskName, args },
+  });
+
 export const mutateMetadataScan = (input: GQL.ScanMetadataInput) =>
   client.mutate<GQL.MetadataScanMutation>({
     mutation: GQL.MetadataScanDocument,

ui/v2.5/src/docs/en/Plugins.md

@@ -0,0 +1,152 @@
+# Plugins
+
+Stash supports the running external tasks via plugins. Plugins are implemented by calling an external binary.
+
+> **⚠️ Note:** Plugin support is still experimental and is likely to change.
+
+# Adding plugins
+
+By default, Stash looks for plugin configurations in the `plugins` sub-directory of the directory where the stash `config.yml` is read. This will either be the `$HOME/.stash` directory or the current working directory.
+
+Plugins are added by adding configuration yaml files (format: `pluginName.yml`) to the `plugins` directory.
+
+Loaded plugins can be viewed in the Plugins page of the Settings. After plugins are added, removed or edited while stash is running, they can be reloaded by clicking `Reload Plugins` button.
+
+# Using plugins
+
+Plugins provide tasks which can be run from the Tasks page. 
+
+> **⚠️ Note:** It is currently only possible to run one task at a time. No queuing is currently implemented.
+
+# Plugin configuration file format
+
+The basic structure of a plugin configuration file is as follows:
+
+```
+name: <plugin name>
+description: <optional description of the plugin>
+version: <optional version tag>
+url: <optional url>
+exec:
+  - <binary name>
+  - <other args...>
+interface: [interface type]
+errLog: [one of none trace, debug, info, warning, error]
+tasks:
+  - ...
+```
+
+## Plugin process execution
+
+The `exec` field is a list with the first element being the binary that will be executed, and the subsequent elements are the arguments passed. The execution process will search the path for the binary, then will attempt to find the program in the same directory as the plugin configuration file. The `exe` extension is not necessary on Windows systems. 
+
+> **⚠️ Note:** The plugin execution process sets the current working directory to that of the stash process.
+
+Arguments can include the plugin's directory with the special string `{pluginDir}`. 
+
+For example, if the plugin executable `my_plugin` is placed in the `plugins` subdirectory and requires arguments `foo` and `bar`, then the `exec` part of the configuration would look like the following:
+
+```
+exec:
+  - my_plugin
+  - foo
+  - bar
+```
+
+Another example might use a python script to execute the plugin. Assuming the python script `foo.py` is placed in the same directory as the plugin config file, the `exec` fragment would look like the following:
+
+```
+exec:
+  - python
+  - {pluginDir}/foo.py
+```
+
+## Plugin interfaces
+
+The `interface` field currently accepts one of two possible values: `rpc` and `raw`. It defaults to `raw` if not provided.
+
+Plugins may log to the stash server by writing to stderr. By default, data written to stderr will be logged by stash at the `error` level. This default behaviour can be changed by setting the `errLog` field.
+
+Plugins can log for specific levels or log progress by prefixing the output string with special control characters. See `pkg/plugin/common/log` for how this is done in go.
+
+### RPC interface
+
+The RPC interface uses JSON-RPC to communicate with the plugin process. A golang plugin utilising the RPC interface is available in the stash source code under `pkg/plugin/examples/gorpc`. RPC plugins are expected to provide an interface that fulfils the `RPCRunner` interface in `pkg/plugin/common`.
+
+RPC plugins are expected to accept requests asynchronously.
+
+When stopping an RPC plugin task, the stash server sends a stop request to the plugin and relies on the plugin to stop itself.
+
+### Raw interface
+
+Raw interface plugins are not required to conform to any particular interface. The stash server will send the plugin input to the plugin process via its stdin stream, encoded as JSON. Raw interface plugins are not required to read the input.
+
+The stash server reads stdout for the plugin's output. If the output can be decoded as a JSON representation of the plugin output data structure then it will do so. If not, it will treat the entire stdout string as the plugin's output.
+
+When stopping a raw plugin task, the stash server kills the spawned process without warning or signals.
+
+## Plugin input
+
+Plugins may accept an input from the stash server. This input is encoded according to the interface, and has the following structure (presented here in JSON format):
+```
+{
+    "server_connection": {
+        "Scheme": "http",
+        "Port": 9999,
+        "SessionCookie": {
+            "Name":"session",
+            "Value":"cookie-value",
+            "Path":"",
+            "Domain":"",
+            "Expires":"0001-01-01T00:00:00Z",
+            "RawExpires":"",
+            "MaxAge":0,
+            "Secure":false,
+            "HttpOnly":false,
+            "SameSite":0,
+            "Raw":"",
+            "Unparsed":null
+        },
+        "Dir": <path to stash config directory>,
+        "PluginDir": <path to plugin config directory>,
+    },
+    "args": {
+        "argKey": "argValue"
+    }
+}
+```
+
+The `server_connection` field contains all the information needed for a plugin to access the parent stash server.
+
+## Plugin output
+
+Plugin output is expected in the following structure (presented here as JSON format):
+
+```
+{
+    "error": <optional error string>
+    "output": <anything>
+}
+```
+
+The `error` field is logged in stash at the `error` log level if present. The `output` is written at the `debug` log level.
+
+## Task configuration
+
+Tasks are configured using the following structure:
+
+```
+tasks:
+  - name: <operation name>
+    description: <optional description>
+    defaultArgs:
+      argKey: argValue
+    execArgs:
+      - <arg to add to the exec line>
+```
+
+A plugin configuration may contain multiple tasks. 
+
+The `defaultArgs` field is used to add inputs to the plugin input sent to the plugin.
+
+The `execArgs` field allows adding extra parameters to the execution arguments for this task.

vendor/github.com/natefinch/pie/.gitignore

@@ -0,0 +1,24 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof

vendor/github.com/natefinch/pie/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT) 
+
+Copyright (c) 2015 Nate Finch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

vendor/github.com/natefinch/pie/README.md

@@ -0,0 +1,183 @@
+# pie [![GoDoc](https://godoc.org/github.com/natefinch/pie?status.svg)](https://godoc.org/github.com/natefinch/pie) [ ![Codeship Status for natefinch/pie](https://app.codeship.com/projects/ea82a1c0-4bae-0135-2de1-02fedcef81c5/status?branch=master)](https://app.codeship.com/projects/232834)
+
+    import "github.com/natefinch/pie"
+
+package pie provides a toolkit for creating plugins for Go applications.
+
+![pie](https://cloud.githubusercontent.com/assets/3185864/7804562/bc35d256-0332-11e5-8562-fe00ec4d10b2.png)
+
+**Why is it called pie?**
+
+Because if you pronounce API like "a pie", then all this consuming and serving
+of APIs becomes a lot more palatable.  Also, pies are the ultimate pluggable
+interface - depending on what's inside, you can get dinner, dessert, a snack, or
+even breakfast.  Plus, then I get to say that plugins in Go are as easy as...
+well, you know.
+
+If you have to explain it to your boss, just say it's an acronym for Plug In
+Executables. <sub>(but it's not, really)</sub>
+
+## About Pie
+
+Plugins using this toolkit and the applications managing those plugins
+communicate via RPC over the plugin application's Stdin and Stdout.
+
+Functions in this package with the prefix `New` are intended to be used by the
+plugin to set up its end of the communication.  Functions in this package
+with the prefix `Start` are intended to be used by the main application to set
+up its end of the communication and start a plugin executable.
+
+<img src="https://cloud.githubusercontent.com/assets/3185864/7915136/8487d69e-0849-11e5-9dfa-13fc868f258f.png" />
+
+This package provides two conceptually different types of plugins, based on
+which side of the communication is the server and which is the client.
+Plugins which provide an API server for the main application to call are
+called Providers.  Plugins which consume an API provided by the main
+application are called Consumers.
+
+The default codec for RPC for this package is Go's gob encoding, however you
+may provide your own codec, such as JSON-RPC provided by net/rpc/jsonrpc.
+
+There is no requirement that plugins for applications using this toolkit be
+written in Go. As long as the plugin application can consume or provide an
+RPC API of the correct codec, it can interoperate with main applications
+using this process.  For example, if your main application uses JSON-RPC,
+many languages are capable of producing an executable that can provide a
+JSON-RPC API for your application to use.
+
+Included in this repo are some simple examples of a master process and a
+plugin process, to see how the library can be used.  An example of the
+standard plugin that provides an API the master process consumes is in the
+examples/provider directory.  master\_provider expects plugin\_provider to be
+in the same directory or in your $PATH.  You can just go install both of
+them, and it'll work correctly.
+
+In addition to a regular plugin that provides an API, this package can be
+used for plugins that consume an API provided by the main process.  To see an
+example of this, look in the examples/consumer folder.
+
+
+## func NewConsumer
+``` go
+func NewConsumer() *rpc.Client
+```
+NewConsumer returns an rpc.Client that will consume an API from the host
+process over this application's Stdin and Stdout using gob encoding.
+
+
+## func NewConsumerCodec
+``` go
+func NewConsumerCodec(f func(io.ReadWriteCloser) rpc.ClientCodec) *rpc.Client
+```
+NewConsumerCodec returns an rpc.Client that will consume an API from the host
+process over this application's Stdin and Stdout using the ClientCodec
+returned by f.
+
+
+## func StartProvider
+``` go
+func StartProvider(output io.Writer, path string, args ...string) (*rpc.Client, error)
+```
+StartProvider start a provider-style plugin application at the given path and
+args, and returns an RPC client that communicates with the plugin using gob
+encoding over the plugin's Stdin and Stdout.  The writer passed to output
+will receive output from the plugin's stderr.  Closing the RPC client
+returned from this function will shut down the plugin application.
+
+
+## func StartProviderCodec
+``` go
+func StartProviderCodec(
+    f func(io.ReadWriteCloser) rpc.ClientCodec,
+    output io.Writer,
+    path string,
+    args ...string,
+) (*rpc.Client, error)
+```
+StartProviderCodec starts a provider-style plugin application at the given
+path and args, and returns an RPC client that communicates with the plugin
+using the ClientCodec returned by f over the plugin's Stdin and Stdout. The
+writer passed to output will receive output from the plugin's stderr.
+Closing the RPC client returned from this function will shut down the plugin
+application.
+
+
+## type Server
+``` go
+type Server struct {
+    // contains filtered or unexported fields
+}
+```
+Server is a type that represents an RPC server that serves an API over
+stdin/stdout.
+
+
+### func NewProvider
+``` go
+func NewProvider() Server
+```
+NewProvider returns a Server that will serve RPC over this
+application's Stdin and Stdout.  This method is intended to be run by the
+plugin application.
+
+
+### func StartConsumer
+``` go
+func StartConsumer(output io.Writer, path string, args ...string) (Server, error)
+```
+StartConsumer starts a consumer-style plugin application with the given path
+and args, writing its stderr to output.  The plugin consumes an API this
+application provides.  The function returns the Server for this host
+application, which should be used to register APIs for the plugin to consume.
+
+
+### func (Server) Close
+``` go
+func (s Server) Close() error
+```
+Close closes the connection with the client.  If the client is a plugin
+process, the process will be stopped.  Further communication using this
+Server will fail.
+
+
+### func (Server) Register
+``` go
+func (s Server) Register(rcvr interface{}) error
+```
+Register publishes in the provider the set of methods of the receiver value
+that satisfy the following conditions:
+
+
+	- exported method
+	- two arguments, both of exported type
+	- the second argument is a pointer
+	- one return value, of type error
+
+It returns an error if the receiver is not an exported type or has no
+suitable methods. It also logs the error using package log. The client
+accesses each method using a string of the form "Type.Method", where Type is
+the receiver's concrete type.
+
+
+### func (Server) RegisterName
+``` go
+func (s Server) RegisterName(name string, rcvr interface{}) error
+```
+RegisterName is like Register but uses the provided name for the type
+instead of the receiver's concrete type.
+
+
+### func (Server) Serve
+``` go
+func (s Server) Serve()
+```
+Serve starts the Server's RPC server, serving via gob encoding.  This call
+will block until the client hangs up.
+
+
+### func (Server) ServeCodec
+``` go
+func (s Server) ServeCodec(f func(io.ReadWriteCloser) rpc.ServerCodec)
+```
+ServeCodec starts the Server's RPC server, serving via the encoding returned
+by f. This call will block until the client hangs up.

vendor/github.com/natefinch/pie/doc.go

@@ -0,0 +1,37 @@
+// Package pie provides a toolkit for creating plugins for Go applications.
+//
+// Plugins using this toolkit and the applications managing those plugins
+// communicate via RPC over the plugin application's Stdin and Stdout.
+//
+// Functions in this package with the prefix New are intended to be used by the
+// plugin to set up its end of the communication.  Functions in this package
+// with the prefix Start are intended to be used by the main application to set
+// up its end of the communication and run a plugin executable.
+//
+// This package provides two conceptually different types of plugins, based on
+// which side of the communication is the server and which is the client.
+// Plugins which provide an API server for the main application to call are
+// called Providers.  Plugins which consume an API provided by the main
+// application are called Consumers.
+//
+// The default codec for RPC for this package is Go's gob encoding, however you
+// may provide your own codec, such as JSON-RPC provided by net/rpc/jsonrpc.
+//
+// There is no requirement that plugins for applications using this toolkit be
+// written in Go. As long as the plugin application can consume or provide an
+// RPC API of the correct codec, it can interoperate with main applications
+// using this process.  For example, if your main application uses JSON-RPC,
+// many languages are capable of producing an executable that can provide a
+// JSON-RPC API for your application to use.
+//
+// Included in this repo are some simple examples of a master process and a
+// plugin process, to see how the library can be used.  An example of the
+// standard plugin that provides an API the master process consumes is in the
+// exmaples/provider directory.  master_provider expects plugin_provider to be
+// in the same directory or in your $PATH.  You can just go install both of
+// them, and it'll work correctly.
+
+// In addition to a regular plugin that provides an API, this package can be
+// used for plugins that consume an API provided by the main process.  To see an
+// example of this, look in the examples/consumer folder.
+package pie

vendor/github.com/natefinch/pie/pie.go

@@ -0,0 +1,260 @@
+package pie
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/rpc"
+	"os"
+	"os/exec"
+	"time"
+)
+
+var errProcStopTimeout = errors.New("process killed after timeout waiting for process to stop")
+
+// NewProvider returns a Server that will serve RPC over this
+// application's Stdin and Stdout.  This method is intended to be run by the
+// plugin application.
+func NewProvider() Server {
+	return Server{
+		server: rpc.NewServer(),
+		rwc:    rwCloser{os.Stdin, os.Stdout},
+	}
+}
+
+// Server is a type that represents an RPC server that serves an API over
+// stdin/stdout.
+type Server struct {
+	server *rpc.Server
+	rwc    io.ReadWriteCloser
+	codec  rpc.ServerCodec
+}
+
+// Close closes the connection with the client.  If the client is a plugin
+// process, the process will be stopped.  Further communication using this
+// Server will fail.
+func (s Server) Close() error {
+	if s.codec != nil {
+		return s.codec.Close()
+	}
+	return s.rwc.Close()
+}
+
+// Serve starts the Server's RPC server, serving via gob encoding.  This call
+// will block until the client hangs up.
+func (s Server) Serve() {
+	s.server.ServeConn(s.rwc)
+}
+
+// ServeCodec starts the Server's RPC server, serving via the encoding returned
+// by f. This call will block until the client hangs up.
+func (s Server) ServeCodec(f func(io.ReadWriteCloser) rpc.ServerCodec) {
+	s.server.ServeCodec(f(s.rwc))
+}
+
+// Register publishes in the provider the set of methods of the receiver value
+// that satisfy the following conditions:
+//
+//	- exported method
+//	- two arguments, both of exported type
+//	- the second argument is a pointer
+//	- one return value, of type error
+//
+// It returns an error if the receiver is not an exported type or has no
+// suitable methods. It also logs the error using package log. The client
+// accesses each method using a string of the form "Type.Method", where Type is
+// the receiver's concrete type.
+func (s Server) Register(rcvr interface{}) error {
+	return s.server.Register(rcvr)
+}
+
+// RegisterName is like Register but uses the provided name for the type
+// instead of the receiver's concrete type.
+func (s Server) RegisterName(name string, rcvr interface{}) error {
+	return s.server.RegisterName(name, rcvr)
+}
+
+// StartProvider start a provider-style plugin application at the given path and
+// args, and returns an RPC client that communicates with the plugin using gob
+// encoding over the plugin's Stdin and Stdout.  The writer passed to output
+// will receive output from the plugin's stderr.  Closing the RPC client
+// returned from this function will shut down the plugin application.
+func StartProvider(output io.Writer, path string, args ...string) (*rpc.Client, error) {
+	pipe, err := start(makeCommand(output, path, args))
+	if err != nil {
+		return nil, err
+	}
+	return rpc.NewClient(pipe), nil
+}
+
+// StartProviderCodec starts a provider-style plugin application at the given
+// path and args, and returns an RPC client that communicates with the plugin
+// using the ClientCodec returned by f over the plugin's Stdin and Stdout. The
+// writer passed to output will receive output from the plugin's stderr.
+// Closing the RPC client returned from this function will shut down the plugin
+// application.
+func StartProviderCodec(
+	f func(io.ReadWriteCloser) rpc.ClientCodec,
+	output io.Writer,
+	path string,
+	args ...string,
+) (*rpc.Client, error) {
+	pipe, err := start(makeCommand(output, path, args))
+	if err != nil {
+		return nil, err
+	}
+	return rpc.NewClientWithCodec(f(pipe)), nil
+}
+
+// StartConsumer starts a consumer-style plugin application with the given path
+// and args, writing its stderr to output.  The plugin consumes an API this
+// application provides.  The function returns the Server for this host
+// application, which should be used to register APIs for the plugin to consume.
+func StartConsumer(output io.Writer, path string, args ...string) (Server, error) {
+	pipe, err := start(makeCommand(output, path, args))
+	if err != nil {
+		return Server{}, err
+	}
+	return Server{
+		server: rpc.NewServer(),
+		rwc:    pipe,
+	}, nil
+}
+
+// NewConsumer returns an rpc.Client that will consume an API from the host
+// process over this application's Stdin and Stdout using gob encoding.
+func NewConsumer() *rpc.Client {
+	return rpc.NewClient(rwCloser{os.Stdin, os.Stdout})
+}
+
+// NewConsumerCodec returns an rpc.Client that will consume an API from the host
+// process over this application's Stdin and Stdout using the ClientCodec
+// returned by f.
+func NewConsumerCodec(f func(io.ReadWriteCloser) rpc.ClientCodec) *rpc.Client {
+	return rpc.NewClientWithCodec(f(rwCloser{os.Stdin, os.Stdout}))
+}
+
+// start runs the plugin and returns an ioPipe that can be used to control the
+// plugin.
+func start(cmd commander) (_ ioPipe, err error) {
+	in, err := cmd.StdinPipe()
+	if err != nil {
+		return ioPipe{}, err
+	}
+	defer func() {
+		if err != nil {
+			in.Close()
+		}
+	}()
+	out, err := cmd.StdoutPipe()
+	if err != nil {
+		return ioPipe{}, err
+	}
+	defer func() {
+		if err != nil {
+			out.Close()
+		}
+	}()
+
+	proc, err := cmd.Start()
+	if err != nil {
+		return ioPipe{}, err
+	}
+	return ioPipe{out, in, proc}, nil
+}
+
+// makeCommand is a function that just creates an exec.Cmd and the process in
+// it. It exists to facilitate testing.
+var makeCommand = func(w io.Writer, path string, args []string) commander {
+	cmd := exec.Command(path, args...)
+	cmd.Stderr = w
+	return execCmd{cmd}
+}
+
+type execCmd struct {
+	*exec.Cmd
+}
+
+func (e execCmd) Start() (osProcess, error) {
+	if err := e.Cmd.Start(); err != nil {
+		return nil, err
+	}
+	return e.Cmd.Process, nil
+}
+
+// commander is an interface that is fulfilled by exec.Cmd and makes our testing
+// a little easier.
+type commander interface {
+	StdinPipe() (io.WriteCloser, error)
+	StdoutPipe() (io.ReadCloser, error)
+	// Start is like exec.Cmd's start, except it also returns the os.Process if
+	// start succeeds.
+	Start() (osProcess, error)
+}
+
+// osProcess is an interface that is fullfilled by *os.Process and makes our
+// testing a little easier.
+type osProcess interface {
+	Wait() (*os.ProcessState, error)
+	Kill() error
+	Signal(os.Signal) error
+}
+
+// ioPipe simply wraps a ReadCloser, WriteCloser, and a Process, and coordinates
+// them so they all close together.
+type ioPipe struct {
+	io.ReadCloser
+	io.WriteCloser
+	proc osProcess
+}
+
+// Close closes the pipe's WriteCloser, ReadClosers, and process.
+func (iop ioPipe) Close() error {
+	err := iop.ReadCloser.Close()
+	if writeErr := iop.WriteCloser.Close(); writeErr != nil {
+		err = writeErr
+	}
+	if procErr := iop.closeProc(); procErr != nil {
+		err = procErr
+	}
+	return err
+}
+
+// procTimeout is the timeout to wait for a process to stop after being
+// signalled.  It is adjustable to keep tests fast.
+var procTimeout = time.Second
+
+// closeProc sends an interrupt signal to the pipe's process, and if it doesn't
+// respond in one second, kills the process.
+func (iop ioPipe) closeProc() error {
+	result := make(chan error, 1)
+	go func() { _, err := iop.proc.Wait(); result <- err }()
+	if err := iop.proc.Signal(os.Interrupt); err != nil {
+		return err
+	}
+	select {
+	case err := <-result:
+		return err
+	case <-time.After(procTimeout):
+		if err := iop.proc.Kill(); err != nil {
+			return fmt.Errorf("error killing process after timeout: %s", err)
+		}
+		return errProcStopTimeout
+	}
+}
+
+// rwCloser just merges a ReadCloser and a WriteCloser into a ReadWriteCloser.
+type rwCloser struct {
+	io.ReadCloser
+	io.WriteCloser
+}
+
+// Close closes both the ReadCloser and the WriteCloser, returning the last
+// error from either.
+func (rw rwCloser) Close() error {
+	err := rw.ReadCloser.Close()
+	if err := rw.WriteCloser.Close(); err != nil {
+		return err
+	}
+	return err
+}

vendor/modules.txt

@@ -200,6 +200,8 @@ github.com/mitchellh/mapstructure
 github.com/modern-go/concurrent
 # github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742
 github.com/modern-go/reflect2
+# github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007
+github.com/natefinch/pie
 # github.com/pelletier/go-toml v1.2.0
 github.com/pelletier/go-toml
 # github.com/pkg/errors v0.8.1