Javascript scraper postprocess (#4200)

* Add javascript post-process action
* Add documentation

pkg/scraper/mapped.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/robertkrimen/otto"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
@@ -523,6 +524,31 @@ func (p *postProcessLbToKg) Apply(ctx context.Context, value string, q mappedQue
 	return value
 }
 
+type postProcessJavascript string
+
+func (p *postProcessJavascript) Apply(ctx context.Context, value string, q mappedQuery) string {
+	vm := otto.New()
+	if err := vm.Set("value", value); err != nil {
+		logger.Warnf("javascript failed to set value: %v", err)
+		return value
+	}
+
+	script, err := vm.Compile("", "(function() { "+string(*p)+"})()")
+	if err != nil {
+		logger.Warnf("javascript failed to compile: %v", err)
+		return value
+	}
+
+	output, err := vm.Run(script)
+	if err != nil {
+		logger.Warnf("javascript failed to run: %v", err)
+		return value
+	}
+
+	// assume output is string
+	return output.String()
+}
+
 type mappedPostProcessAction struct {
 	ParseDate    string                   `yaml:"parseDate"`
 	SubtractDays bool                     `yaml:"subtractDays"`
@@ -531,65 +557,75 @@ type mappedPostProcessAction struct {
 	Map          map[string]string        `yaml:"map"`
 	FeetToCm     bool                     `yaml:"feetToCm"`
 	LbToKg       bool                     `yaml:"lbToKg"`
+	Javascript   string                   `yaml:"javascript"`
 }
 
 func (a mappedPostProcessAction) ToPostProcessAction() (postProcessAction, error) {
 	var found string
 	var ret postProcessAction
 
+	ensureOnly := func(field string) error {
+		if found != "" {
+			return fmt.Errorf("post-process actions must have a single field, found %s and %s", found, field)
+		}
+		found = field
+		return nil
+	}
+
 	if a.ParseDate != "" {
 		found = "parseDate"
 		action := postProcessParseDate(a.ParseDate)
 		ret = &action
 	}
 	if len(a.Replace) > 0 {
-		if found != "" {
+		if err := ensureOnly("replace"); err != nil {
-			return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "replace")
+			return nil, err
 		}
-		found = "replace"
 		action := postProcessReplace(a.Replace)
 		ret = &action
 	}
 	if a.SubScraper != nil {
-		if found != "" {
+		if err := ensureOnly("subScraper"); err != nil {
-			return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "subScraper")
+			return nil, err
 		}
-		found = "subScraper"
 		action := postProcessSubScraper(*a.SubScraper)
 		ret = &action
 	}
 	if a.Map != nil {
-		if found != "" {
+		if err := ensureOnly("map"); err != nil {
-			return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "map")
+			return nil, err
 		}
-		found = "map"
 		action := postProcessMap(a.Map)
 		ret = &action
 	}
 	if a.FeetToCm {
-		if found != "" {
+		if err := ensureOnly("feetToCm"); err != nil {
-			return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "feetToCm")
+			return nil, err
 		}
-		found = "feetToCm"
 		action := postProcessFeetToCm(a.FeetToCm)
 		ret = &action
 	}
 	if a.LbToKg {
-		if found != "" {
+		if err := ensureOnly("lbToKg"); err != nil {
-			return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "lbToKg")
+			return nil, err
 		}
-		found = "lbToKg"
 		action := postProcessLbToKg(a.LbToKg)
 		ret = &action
 	}
 	if a.SubtractDays {
-		if found != "" {
+		if err := ensureOnly("subtractDays"); err != nil {
-			return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "subtractDays")
+			return nil, err
 		}
-		// found = "subtractDays"
 		action := postProcessSubtractDays(a.SubtractDays)
 		ret = &action
 	}
+	if a.Javascript != "" {
+		if err := ensureOnly("javascript"); err != nil {
+			return nil, err
+		}
+		action := postProcessJavascript(a.Javascript)
+		ret = &action
+	}
 
 	if ret == nil {
 		return nil, errors.New("invalid post-process action")

ui/v2.5/src/docs/en/Manual/ScraperDevelopment.md

@@ -341,6 +341,20 @@ scene:
 ### Post-processing options
 
 Post-processing operations are contained in the `postProcess` key. Post-processing operations are performed in the order they are specified. The following post-processing operations are available:
+* `javascript`: accepts a javascript code block, that must return a string value. The input string is declared in the `value` variable. If an error occurs while compiling or running the script, then the original value is returned.
+Example:
+```yaml
+performer:
+  Name:
+    selector: //div[@class="example element"]
+    postProcess:
+      - javascript: |
+          // capitalise the first letter
+          if (value && value.length) {
+            return value[0].toUpperCase() + value.substring(1)
+          }
+```
+Note that the `otto` javascript engine is missing a few built-in methods and may not be consistent with other modern javascript implementations.
 * `feetToCm`: converts a string containing feet and inches numbers into centimeters. Looks for up to two separate integers and interprets the first as the number of feet, and the second as the number of inches. The numbers can be separated by any non-numeric character including the `.` character. It does not handle decimal numbers. For example `6.3` and `6ft3.3` would both be interpreted as 6 feet, 3 inches before converting into centimeters.
 * `lbToKg`: converts a string containing lbs to kg.
 * `map`: contains a map of input values to output values. Where a value matches one of the input values, it is replaced with the matching output value. If no value is matched, then value is unmodified.