Add JSON scrape support (#717)

* Add support for scene fragment scrape in xpath
2025-12-16 05:13:46 +01:00 · 2020-08-10 14:21:50 +10:00 · 2020-08-10 14:21:50 +10:00 · 7158e83b75
commit 7158e83b75
parent 470a2b5833
28 changed files with 5005 additions and 14 deletions
--- a/go.mod
+++ b/go.mod
@ -26,6 +26,7 @@ require (
 	github.com/spf13/pflag v1.0.3
 	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.5.1
+	github.com/tidwall/gjson v1.6.0
 	github.com/vektah/gqlparser v1.1.2
 	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
 	golang.org/x/image v0.0.0-20190118043309-183bebdce1b2
--- a/go.sum
+++ b/go.sum
@ -609,7 +609,13 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
+github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc=
+github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
 github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
--- a/pkg/scraper/action.go
+++ b/pkg/scraper/action.go
@ -8,17 +8,19 @@ const (
 	scraperActionScript scraperAction = "script"
 	scraperActionStash  scraperAction = "stash"
 	scraperActionXPath  scraperAction = "scrapeXPath"
+	scraperActionJson   scraperAction = "scrapeJson"
 )

 var allScraperAction = []scraperAction{
 	scraperActionScript,
 	scraperActionStash,
 	scraperActionXPath,
+	scraperActionJson,
 }

 func (e scraperAction) IsValid() bool {
 	switch e {
-	case scraperActionScript, scraperActionStash, scraperActionXPath:
+	case scraperActionScript, scraperActionStash, scraperActionXPath, scraperActionJson:
 		return true
 	}
 	return false
@ -47,6 +49,8 @@ func getScraper(scraper scraperTypeConfig, config config, globalConfig GlobalCon
 		return newStashScraper(scraper, config, globalConfig)
 	case scraperActionXPath:
 		return newXpathScraper(scraper, config, globalConfig)
+	case scraperActionJson:
+		return newJsonScraper(scraper, config, globalConfig)
 	}

 	panic("unknown scraper action: " + scraper.Action)
--- a/pkg/scraper/config.go
+++ b/pkg/scraper/config.go
@ -44,6 +44,9 @@ type config struct {
 	// Xpath scraping configurations
 	XPathScrapers mappedScrapers `yaml:"xPathScrapers"`

+	// Json scraping configurations
+	JsonScrapers mappedScrapers `yaml:"jsonScrapers"`
+
 	// Scraping driver options
 	DriverOptions *scraperDriverOptions `yaml:"driver"`
 }
--- a/pkg/scraper/json.go
+++ b/pkg/scraper/json.go
@ -0,0 +1,191 @@
+package scraper
+
+import (
+	"errors"
+	"io/ioutil"
+	"net/url"
+	"strings"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/tidwall/gjson"
+)
+
+type jsonScraper struct {
+	scraper      scraperTypeConfig
+	config       config
+	globalConfig GlobalConfig
+}
+
+func newJsonScraper(scraper scraperTypeConfig, config config, globalConfig GlobalConfig) *jsonScraper {
+	return &jsonScraper{
+		scraper:      scraper,
+		config:       config,
+		globalConfig: globalConfig,
+	}
+}
+
+func (s *jsonScraper) getJsonScraper() *mappedScraper {
+	return s.config.JsonScrapers[s.scraper.Scraper]
+}
+
+func (s *jsonScraper) scrapeURL(url string) (string, *mappedScraper, error) {
+	scraper := s.getJsonScraper()
+
+	if scraper == nil {
+		return "", nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config")
+	}
+
+	doc, err := s.loadURL(url)
+
+	if err != nil {
+		return "", nil, err
+	}
+
+	return doc, scraper, nil
+}
+
+func (s *jsonScraper) loadURL(url string) (string, error) {
+	r, err := loadURL(url, s.config, s.globalConfig)
+	if err != nil {
+		return "", err
+	}
+
+	doc, err := ioutil.ReadAll(r)
+	if err != nil {
+		return "", err
+	}
+
+	docStr := string(doc)
+	if !gjson.Valid(docStr) {
+		return "", errors.New("not valid json")
+	}
+
+	if err == nil && s.config.DebugOptions != nil && s.config.DebugOptions.PrintHTML {
+		logger.Infof("loadURL (%s) response: \n%s", url, docStr)
+	}
+
+	return docStr, err
+}
+
+func (s *jsonScraper) scrapePerformerByURL(url string) (*models.ScrapedPerformer, error) {
+	doc, scraper, err := s.scrapeURL(url)
+	if err != nil {
+		return nil, err
+	}
+
+	q := s.getJsonQuery(doc)
+	return scraper.scrapePerformer(q)
+}
+
+func (s *jsonScraper) scrapeSceneByURL(url string) (*models.ScrapedScene, error) {
+	doc, scraper, err := s.scrapeURL(url)
+	if err != nil {
+		return nil, err
+	}
+
+	q := s.getJsonQuery(doc)
+	return scraper.scrapeScene(q)
+}
+
+func (s *jsonScraper) scrapePerformersByName(name string) ([]*models.ScrapedPerformer, error) {
+	scraper := s.getJsonScraper()
+
+	if scraper == nil {
+		return nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config")
+	}
+
+	const placeholder = "{}"
+
+	// replace the placeholder string with the URL-escaped name
+	escapedName := url.QueryEscape(name)
+
+	url := s.scraper.QueryURL
+	url = strings.Replace(url, placeholder, escapedName, -1)
+
+	doc, err := s.loadURL(url)
+
+	if err != nil {
+		return nil, err
+	}
+
+	q := s.getJsonQuery(doc)
+	return scraper.scrapePerformers(q)
+}
+
+func (s *jsonScraper) scrapePerformerByFragment(scrapedPerformer models.ScrapedPerformerInput) (*models.ScrapedPerformer, error) {
+	return nil, errors.New("scrapePerformerByFragment not supported for json scraper")
+}
+
+func (s *jsonScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) {
+	storedScene, err := sceneFromUpdateFragment(scene)
+	if err != nil {
+		return nil, err
+	}
+
+	if storedScene == nil {
+		return nil, errors.New("no scene found")
+	}
+
+	// construct the URL
+	url := constructSceneURL(s.scraper.QueryURL, storedScene)
+
+	scraper := s.getJsonScraper()
+
+	if scraper == nil {
+		return nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config")
+	}
+
+	doc, err := s.loadURL(url)
+
+	if err != nil {
+		return nil, err
+	}
+
+	q := s.getJsonQuery(doc)
+	return scraper.scrapeScene(q)
+}
+
+func (s *jsonScraper) getJsonQuery(doc string) *jsonQuery {
+	return &jsonQuery{
+		doc:     doc,
+		scraper: s,
+	}
+}
+
+type jsonQuery struct {
+	doc     string
+	scraper *jsonScraper
+}
+
+func (q *jsonQuery) runQuery(selector string) []string {
+	value := gjson.Get(q.doc, selector)
+
+	if !value.Exists() {
+		logger.Warnf("Could not find json path '%s' in json object", selector)
+		return nil
+	}
+
+	var ret []string
+	if value.IsArray() {
+		value.ForEach(func(k, v gjson.Result) bool {
+			ret = append(ret, v.String())
+			return true
+		})
+	} else {
+		ret = append(ret, value.String())
+	}
+
+	return ret
+}
+
+func (q *jsonQuery) subScrape(value string) mappedQuery {
+	doc, err := q.scraper.loadURL(value)
+
+	if err != nil {
+		logger.Warnf("Error getting URL '%s' for sub-scraper: %s", value, err.Error())
+		return nil
+	}
+
+	return q.scraper.getJsonQuery(doc)
+}
--- a/pkg/scraper/json_test.go
+++ b/pkg/scraper/json_test.go
@ -0,0 +1,93 @@
+package scraper
+
+import (
+	"testing"
+
+	"gopkg.in/yaml.v2"
+)
+
+func TestJsonPerformerScraper(t *testing.T) {
+	const yamlStr = `name: Test
+jsonScrapers:
+  performerScraper:
+    common:
+      $extras: data.extras
+    performer:
+      Name: data.name
+      Gender: $extras.gender
+      Birthdate: $extras.birthday
+      Ethnicity: $extras.ethnicity
+      Height: $extras.height
+      Measurements: $extras.measurements
+      Tattoos: $extras.tattoos
+      Piercings: $extras.piercings
+      Aliases: data.aliases
+      Image: data.image
+`
+
+	const json = `
+{
+	"data": {
+        "id": "2cd4146b-637d-49b1-8ff9-19d4a06947bb",
+        "name": "Mia Malkova",
+        "bio": "Some girls are so damn hot that they can get you bent out of shape, and you will not even be mad at them for doing so. Well, tawny blonde Mia Malkova can bend her body into any shape she pleases, and that’s sure to satisfy all of the horny cocks and wet pussies out there. This girl has acrobatic and contortionist abilities that could even twist a pretzel into a new knot, which can be very helpful in the ... arrow_drop_down Some girls are so damn hot that they can get you bent out of shape, and you will not even be mad at them for doing so. Well, tawny blonde Mia Malkova can bend her body into any shape she pleases, and that’s sure to satisfy all of the horny cocks and wet pussies out there. This girl has acrobatic and contortionist abilities that could even twist a pretzel into a new knot, which can be very helpful in the VR Porn movies – trust us. Ankles behind her neck and feet over her back so she can kiss her toes, turned, twisted and gyrating, she can fuck any which way she wants (and that ass!), will surely make you fall in love with this hot Virtual Reality Porn slut, as she is one of the finest of them all. Talking about perfection, maybe it’s all the acrobatic work that keeps it in such gorgeous shape? Who cares really, because you just want to take a big bite out of it and never let go. But it’s not all about the body. Mia’s also got a great smile, which might not sound kinky, but believe us, it is a smile that will heat up your innards and drop your pants. Is it her golden skin, her innocent pink lips or that heart-shaped face? There is just too much good stuff going on with Mia Malkova, which is maybe why these past few years have heaped awards upon awards on this Southern California native. Mia came to VR Bangers for her first VR Porn video, so you know she’s only going for top-notch scenes with top-game performers, men, and women. Better hit up that yoga studio if you ever dream of being able to bang a flexible and talented chick like lady Malkova. arrow_drop_up",
+        "extras": {
+            "gender": "Female",
+            "birthday": "1992-07-01",
+            "birthday_timestamp": 709948800,
+            "birthplace": "Palm Springs, California, United States",
+            "active": 1,
+            "astrology": "Cancer (Jun 21 - Jul 22)",
+            "ethnicity": "Caucasian",
+            "nationality": "United States",
+            "hair_colour": "Blonde",
+            "weight": "126 lbs (or 57 kg)",
+            "height": "5'6\" (or 167 cm)",
+            "measurements": "34-26-36",
+            "cupsize": "34C (75C)",
+            "tattoos": "None",
+            "piercings": "Navel",
+            "first_seen": null
+        },
+        "aliases": [
+            "Mia Bliss",
+            "Madison Clover",
+            "Madison Swan",
+            "Mia Mountain",
+            "Mia M.",
+            "Mia Malvoka",
+            "Mia Molkova",
+            "Mia Thomas"
+        ],
+		"image": "https:\/\/thumb.metadataapi.net\/unsafe\/1000x1500\/smart\/filters:sharpen():upscale()\/https%3A%2F%2Fcdn.metadataapi.net%2Fperformer%2F49%2F05%2F30%2Fade2255dc065032a89ebb23f0e038fa%2Fposter%2Fmia-malkova.jpg%3Fid1582610531"
+	}
+}
+`
+
+	c := &config{}
+	err := yaml.Unmarshal([]byte(yamlStr), &c)
+
+	if err != nil {
+		t.Fatalf("Error loading yaml: %s", err.Error())
+	}
+
+	// perform scrape using json string
+	performerScraper := c.JsonScrapers["performerScraper"]
+
+	q := &jsonQuery{
+		doc: json,
+	}
+
+	scrapedPerformer, err := performerScraper.scrapePerformer(q)
+	if err != nil {
+		t.Fatalf("Error scraping performer: %s", err.Error())
+	}
+
+	verifyField(t, "Mia Malkova", scrapedPerformer.Name, "Name")
+	verifyField(t, "Female", scrapedPerformer.Gender, "Gender")
+	verifyField(t, "1992-07-01", scrapedPerformer.Birthdate, "Birthdate")
+	verifyField(t, "Caucasian", scrapedPerformer.Ethnicity, "Ethnicity")
+	verifyField(t, "5'6\" (or 167 cm)", scrapedPerformer.Height, "Height")
+	verifyField(t, "None", scrapedPerformer.Tattoos, "Tattoos")
+	verifyField(t, "Navel", scrapedPerformer.Piercings, "Piercings")
+}
--- a/pkg/scraper/stash.go
+++ b/pkg/scraper/stash.go
@ -191,3 +191,14 @@ func (s *stashScraper) scrapePerformerByURL(url string) (*models.ScrapedPerforme
 func (s *stashScraper) scrapeSceneByURL(url string) (*models.ScrapedScene, error) {
 	return nil, errors.New("scrapeSceneByURL not supported for stash scraper")
 }
+
+func sceneFromUpdateFragment(scene models.SceneUpdateInput) (*models.Scene, error) {
+	qb := models.NewSceneQueryBuilder()
+	id, err := strconv.Atoi(scene.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO - should we modify it with the input?
+	return qb.Find(id)
+}
--- a/pkg/scraper/url.go
+++ b/pkg/scraper/url.go
@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"

@ -18,10 +19,25 @@ import (
 	"github.com/chromedp/chromedp"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
 	"golang.org/x/net/html/charset"
 	"golang.org/x/net/publicsuffix"
 )

+// Timeout for the scrape http request. Includes transfer time. May want to make this
+// configurable at some point.
+const scrapeGetTimeout = time.Second * 30
+
+func constructSceneURL(url string, scene *models.Scene) string {
+	// support checksum, title and filename
+	ret := strings.Replace(url, "{checksum}", scene.Checksum.String, -1)
+	ret = strings.Replace(url, "{oshash}", scene.OSHash.String, -1)
+	ret = strings.Replace(ret, "{filename}", filepath.Base(scene.Path), -1)
+	ret = strings.Replace(ret, "{title}", scene.Title.String, -1)
+
+	return ret
+}
+
 func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Reader, error) {
 	driverOptions := scraperConfig.DriverOptions
 	if driverOptions != nil && driverOptions.UseCDP {
--- a/pkg/scraper/xpath.go
+++ b/pkg/scraper/xpath.go
@ -6,7 +6,6 @@ import (
 	"net/url"
 	"regexp"
 	"strings"
-	"time"

 	"github.com/antchfx/htmlquery"

@ -16,10 +15,6 @@ import (
 	"github.com/stashapp/stash/pkg/models"
 )

-// Timeout for the scrape http request. Includes transfer time. May want to make this
-// configurable at some point.
-const scrapeGetTimeout = time.Second * 30
-
 type xpathScraper struct {
 	scraper      scraperTypeConfig
 	config       config
@ -104,7 +99,32 @@ func (s *xpathScraper) scrapePerformerByFragment(scrapedPerformer models.Scraped
 }

 func (s *xpathScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) {
-	return nil, errors.New("scrapeSceneByFragment not supported for xpath scraper")
+	storedScene, err := sceneFromUpdateFragment(scene)
+	if err != nil {
+		return nil, err
+	}
+
+	if storedScene == nil {
+		return nil, errors.New("no scene found")
+	}
+
+	// construct the URL
+	url := constructSceneURL(s.scraper.QueryURL, storedScene)
+
+	scraper := s.getXpathScraper()
+
+	if scraper == nil {
+		return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config")
+	}
+
+	doc, err := s.loadURL(url)
+
+	if err != nil {
+		return nil, err
+	}
+
+	q := s.getXPathQuery(doc)
+	return scraper.scrapeScene(q)
 }

 func (s *xpathScraper) loadURL(url string) (*html.Node, error) {
--- a/ui/v2.5/src/components/Changelog/versions/v030.tsx
+++ b/ui/v2.5/src/components/Changelog/versions/v030.tsx
@ -5,6 +5,7 @@ const markup = `
 #### 💥 **Note: After upgrading, the next scan will populate all scenes with oshash hashes. MD5 calculation can be disabled after populating the oshash for all scenes. See \`Hashing Algorithms\` in the \`Configuration\` section of the manual for details. **

 ### ✨ New Features
+*  Add support for JSON scrapers.
 *  Add support for plugin tasks.
 *  Add oshash algorithm for hashing scene video files. Enabled by default on new systems.
 *  Support (re-)generation of generated content for specific scenes.
--- a/ui/v2.5/src/docs/en/Scraping.md
+++ b/ui/v2.5/src/docs/en/Scraping.md
@ -170,7 +170,15 @@ sceneByURL:

 The above configuration requires that `sceneScraper` exists in the `xPathScrapers` configuration.

-#### Use with `performerByName`
+XPath scraping configurations specify the mapping between object fields and an xpath selector. The xpath scraper scrapes the applicable URL and uses xpath to populate the object fields.
+
+### scrapeJson
+
+This action works in the same way as `scrapeXPath`, but uses a mapped json configuration to parse. It uses the top-level `jsonScrapers` configuration. This action is valid for `performerByName`, `performerByURL`, `sceneByFragment` and `sceneByURL`.
+
+JSON scraping configurations specify the mapping between object fields and a GJSON selector. The JSON scraper scrapes the applicable URL and uses [GJSON](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) to parse the returned JSON object and populate the object fields.
+
+### scrapeXPath and scrapeJson use with `performerByName`

 For `performerByName`, the `queryURL` field must be present also. This field is used to perform a search query URL for performer names. The placeholder string sequence `{}` is replaced with the performer name search string. For the subsequent performer scrape to work, the `URL` field must be filled in with the URL of the performer page that matches a URL given in a `performerByURL` scraping configuration. For example:

@ -194,13 +202,36 @@ xPathScrapers:
    # ... performer scraper details ...
 ```

-#### XPath scrapers configuration
+### scrapeXPath and scrapeJson use with `sceneByFragment`

-The top-level `xPathScrapers` field contains xpath scraping configurations, freely named. The scraping configuration may contain a `common` field, and must contain `performer` or `scene` depending on the scraping type it is configured for. 
+For `sceneByFragment`, the `queryURL` field must also be present. This field is used to build a query URL for scenes. For `sceneByFragment`, the `queryURL` field supports the following placeholder fields:
+* `{checksum}` - the MD5 checksum of the scene
+* `{oshash}` - the oshash of the scene
+* `{filename}` - the base filename of the scene
+* `{title}` - the title of the scene
+
+For example:
+
+```
+sceneByFragment:
+  action: scrapeJson
+  queryURL: https://metadataapi.net/api/scenes?parse={filename}&limit=1
+  scraper: sceneQueryScraper
+```
+
+### Xpath and JSON scrapers configuration
+
+The top-level `xPathScrapers` field contains xpath scraping configurations, freely named. These are referenced in the `scraper` field for `scrapeXPath` scrapers. 
+
+Likewise, the top-level `jsonScrapers` field contains json scraping configurations.
+
+Collectively, these configurations are known as mapped scraping configurations. 
+
+A mapped scraping configuration may contain a `common` field, and must contain `performer` or `scene` depending on the scraping type it is configured for. 

 Within the `performer`/`scene` field are key/value pairs corresponding to the golang fields (see below) on the performer/scene object. These fields are case-sensitive. 

-The values of these may be either a simple xpath value, which tells the system where to get the value of the field from, or a more advanced configuration (see below). For example:
+The values of these may be either a simple selector value, which tells the system where to get the value of the field from, or a more advanced configuration (see below). For example, for an xpath configuration:

 ```
 performer:
@ -209,7 +240,14 @@ performer:

 This will set the `Name` attribute of the returned performer to the text content of the element that matches `<h1 itemprop="name">...`.

-The value may also be a sub-object. If it is a sub-object, then the xpath must be set to the `selector` key of the sub-object. For example, using the same xpath as above:
+For a json configuration:
+
+```
+performer:
+  Name: data.name
+```
+
+The value may also be a sub-object. If it is a sub-object, then the selector must be set to the `selector` key of the sub-object. For example, using the same xpath as above:

 ```
 performer:
@ -231,7 +269,7 @@ performer:

 ##### Common fragments

-The `common` field is used to configure xpath fragments that can be referenced in the xpath strings. These are key-value pairs where the key is the string to reference the fragment, and the value is the string that the fragment will be replaced with. For example:
+The `common` field is used to configure selector fragments that can be referenced in the selector strings. These are key-value pairs where the key is the string to reference the fragment, and the value is the string that the fragment will be replaced with. For example:

 ```
 common:
@ -299,7 +337,7 @@ When `useCDP` is set to true, stash will execute or connect to an instance of Ch

 `Chrome CDP path` can be set to a path to the chrome executable, or an http(s) address to remote chrome instance (for example: `http://localhost:9222/json/version`).

-##### Example
+##### XPath scraper example

 A performer and scene xpath scraper is shown as an example below:

@ -361,11 +399,98 @@ xPathScrapers:

 See also [#333](https://github.com/stashapp/stash/pull/333) for more examples.

+##### JSON scraper example
+
+A performer and scene scraper for ThePornDB is shown below:
+
+```
+name: ThePornDB
+performerByName:
+  action: scrapeJson
+  queryURL: https://metadataapi.net/api/performers?q={}
+  scraper: performerSearch
+performerByURL:
+  - action: scrapeJson
+    url:
+      - https://metadataapi.net/api/performers/
+    scraper: performerScraper
+sceneByURL:
+  - action: scrapeJson
+    url:
+      - https://metadataapi.net/api/scenes/
+    scraper: sceneScraper
+sceneByFragment:
+  action: scrapeJson
+  queryURL: https://metadataapi.net/api/scenes?parse={filename}&limit=1
+  scraper: sceneQueryScraper
+jsonScrapers:
+  performerSearch:
+    performer:
+      Name: data.#.name
+      URL:
+        selector: data.#.id
+        replace:
+          - regex: ^
+            with: https://metadataapi.net/api/performers/
+
+  performerScraper:
+    common:
+      $extras: data.extras
+    performer:
+      Name: data.name
+      Gender: $extras.gender
+      Birthdate: $extras.birthday
+      Ethnicity: $extras.ethnicity
+      Height: $extras.height
+      Measurements: $extras.measurements
+      Tattoos: $extras.tattoos
+      Piercings: $extras.piercings
+      Aliases: data.aliases
+      Image: data.image
+
+  sceneScraper:
+    common:
+      $performers: data.performers
+    scene:
+      Title: data.title
+      Details: data.description
+      Date: data.date
+      URL: data.url
+      Image: data.background.small
+      Performers:
+        Name: data.performers.#.name
+      Studio:
+        Name: data.site.name
+      Tags:
+        Name: data.tags.#.tag
+  
+  sceneQueryScraper:
+    common:
+      $data: data.0
+      $performers: data.0.performers
+    scene:
+      Title: $data.title
+      Details: $data.description
+      Date: $data.date
+      URL: $data.url
+      Image: $data.background.small
+      Performers:
+        Name: $data.performers.#.name
+      Studio:
+        Name: $data.site.name
+      Tags:
+        Name: $data.tags.#.tag  
+```
+
 #### XPath resources:

 - Test XPaths in Firefox: https://addons.mozilla.org/en-US/firefox/addon/try-xpath/
 - XPath cheatsheet: https://devhints.io/xpath

+#### GJSON resources:
+
+- GJSON Path Syntax: https://github.com/tidwall/gjson/blob/master/SYNTAX.md
+
 #### Object fields
 ##### Performer

--- a/vendor/github.com/tidwall/gjson/.travis.yml
+++ b/vendor/github.com/tidwall/gjson/.travis.yml
@ -0,0 +1 @@
+language: go
--- a/vendor/github.com/tidwall/gjson/LICENSE
+++ b/vendor/github.com/tidwall/gjson/LICENSE
@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Josh Baker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--- a/vendor/github.com/tidwall/gjson/README.md
+++ b/vendor/github.com/tidwall/gjson/README.md
@ -0,0 +1,495 @@
+<p align="center">
+<img 
+    src="logo.png" 
+    width="240" height="78" border="0" alt="GJSON">
+<br>
+<a href="https://travis-ci.org/tidwall/gjson"><img src="https://img.shields.io/travis/tidwall/gjson.svg?style=flat-square" alt="Build Status"></a>
+<a href="https://godoc.org/github.com/tidwall/gjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
+<a href="http://tidwall.com/gjson-play"><img src="https://img.shields.io/badge/%F0%9F%8F%90-playground-9900cc.svg?style=flat-square" alt="GJSON Playground"></a>
+</p>
+
+
+
+<p align="center">get json values quickly</a></p>
+
+GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document.
+It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array), and [parsing json lines](#json-lines).
+
+Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool.
+
+Getting Started
+===============
+
+## Installing
+
+To start using GJSON, install Go and run `go get`:
+
+```sh
+$ go get -u github.com/tidwall/gjson
+```
+
+This will retrieve the library.
+
+## Get a value
+Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". When the value is found it's returned immediately. 
+
+```go
+package main
+
+import "github.com/tidwall/gjson"
+
+const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}`
+
+func main() {
+	value := gjson.Get(json, "name.last")
+	println(value.String())
+}
+```
+
+This will print:
+
+```
+Prichard
+```
+*There's also the [GetMany](#get-multiple-values-at-once) function to get multiple values at once, and [GetBytes](#working-with-bytes) for working with JSON byte slices.*
+
+## Path Syntax
+
+Below is a quick overview of the path syntax, for more complete information please
+check out [GJSON Syntax](SYNTAX.md).
+
+A path is a series of keys separated by a dot.
+A key may contain special wildcard characters '\*' and '?'.
+To access an array value use the index as the key.
+To get the number of elements in an array or to access a child path, use the '#' character.
+The dot and wildcard characters can be escaped with '\\'.
+
+```json
+{
+  "name": {"first": "Tom", "last": "Anderson"},
+  "age":37,
+  "children": ["Sara","Alex","Jack"],
+  "fav.movie": "Deer Hunter",
+  "friends": [
+    {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]},
+    {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]},
+    {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]}
+  ]
+}
+```
+```
+"name.last"          >> "Anderson"
+"age"                >> 37
+"children"           >> ["Sara","Alex","Jack"]
+"children.#"         >> 3
+"children.1"         >> "Alex"
+"child*.2"           >> "Jack"
+"c?ildren.0"         >> "Sara"
+"fav\.movie"         >> "Deer Hunter"
+"friends.#.first"    >> ["Dale","Roger","Jane"]
+"friends.1.last"     >> "Craig"
+```
+
+You can also query an array for the first match by using `#(...)`, or find all 
+matches with `#(...)#`. Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` 
+comparison operators and the simple pattern matching `%` (like) and `!%` 
+(not like) operators.
+
+```
+friends.#(last=="Murphy").first    >> "Dale"
+friends.#(last=="Murphy")#.first   >> ["Dale","Jane"]
+friends.#(age>45)#.last            >> ["Craig","Murphy"]
+friends.#(first%"D*").last         >> "Murphy"
+friends.#(first!%"D*").last        >> "Craig"
+friends.#(nets.#(=="fb"))#.first   >> ["Dale","Roger"]
+```
+
+*Please note that prior to v1.3.0, queries used the `#[...]` brackets. This was
+changed in v1.3.0 as to avoid confusion with the new
+[multipath](SYNTAX.md#multipaths) syntax. For backwards compatibility, 
+`#[...]` will continue to work until the next major release.*
+
+## Result Type
+
+GJSON supports the json types `string`, `number`, `bool`, and `null`. 
+Arrays and Objects are returned as their raw json types. 
+
+The `Result` type holds one of these:
+
+```
+bool, for JSON booleans
+float64, for JSON numbers
+string, for JSON string literals
+nil, for JSON null
+```
+
+To directly access the value:
+
+```go
+result.Type    // can be String, Number, True, False, Null, or JSON
+result.Str     // holds the string
+result.Num     // holds the float64 number
+result.Raw     // holds the raw json
+result.Index   // index of raw value in original json, zero means index unknown
+```
+
+There are a variety of handy functions that work on a result:
+
+```go
+result.Exists() bool
+result.Value() interface{}
+result.Int() int64
+result.Uint() uint64
+result.Float() float64
+result.String() string
+result.Bool() bool
+result.Time() time.Time
+result.Array() []gjson.Result
+result.Map() map[string]gjson.Result
+result.Get(path string) Result
+result.ForEach(iterator func(key, value Result) bool)
+result.Less(token Result, caseSensitive bool) bool
+```
+
+The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types:
+
+The `result.Array()` function returns back an array of values.
+If the result represents a non-existent value, then an empty array will be returned.
+If the result is not a JSON array, the return value will be an array containing one result.
+
+```go
+boolean >> bool
+number  >> float64
+string  >> string
+null    >> nil
+array   >> []interface{}
+object  >> map[string]interface{}
+```
+
+### 64-bit integers
+
+The `result.Int()` and `result.Uint()` calls are capable of reading all 64 bits, allowing for large JSON integers.
+
+```go
+result.Int() int64    // -9223372036854775808 to 9223372036854775807
+result.Uint() int64   // 0 to 18446744073709551615
+```
+
+## Modifiers and path chaining 
+
+New in version 1.2 is support for modifier functions and path chaining.
+
+A modifier is a path component that performs custom processing on the 
+json.
+
+Multiple paths can be "chained" together using the pipe character. 
+This is useful for getting results from a modified query.
+
+For example, using the built-in `@reverse` modifier on the above json document,
+we'll get `children` array and reverse the order:
+
+```
+"children|@reverse"           >> ["Jack","Alex","Sara"]
+"children|@reverse|0"         >> "Jack"
+```
+
+There are currently the following built-in modifiers:
+
+- `@reverse`: Reverse an array or the members of an object.
+- `@ugly`: Remove all whitespace from a json document.
+- `@pretty`: Make the json document more human readable.
+- `@this`: Returns the current element. It can be used to retrieve the root element.
+- `@valid`: Ensure the json document is valid.
+- `@flatten`: Flattens an array.
+- `@join`: Joins multiple objects into a single object.
+
+### Modifier arguments
+
+A modifier may accept an optional argument. The argument can be a valid JSON 
+document or just characters.
+
+For example, the `@pretty` modifier takes a json object as its argument. 
+
+```
+@pretty:{"sortKeys":true} 
+```
+
+Which makes the json pretty and orders all of its keys.
+
+```json
+{
+  "age":37,
+  "children": ["Sara","Alex","Jack"],
+  "fav.movie": "Deer Hunter",
+  "friends": [
+    {"age": 44, "first": "Dale", "last": "Murphy"},
+    {"age": 68, "first": "Roger", "last": "Craig"},
+    {"age": 47, "first": "Jane", "last": "Murphy"}
+  ],
+  "name": {"first": "Tom", "last": "Anderson"}
+}
+```
+
+*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`. 
+Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.*
+
+### Custom modifiers
+
+You can also add custom modifiers.
+
+For example, here we create a modifier that makes the entire json document upper
+or lower case.
+
+```go
+gjson.AddModifier("case", func(json, arg string) string {
+  if arg == "upper" {
+    return strings.ToUpper(json)
+  }
+  if arg == "lower" {
+    return strings.ToLower(json)
+  }
+  return json
+})
+```
+
+```
+"children|@case:upper"           >> ["SARA","ALEX","JACK"]
+"children|@case:lower|@reverse"  >> ["jack","alex","sara"]
+```
+
+## JSON Lines
+
+There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array. 
+
+For example:
+
+```
+{"name": "Gilbert", "age": 61}
+{"name": "Alexa", "age": 34}
+{"name": "May", "age": 57}
+{"name": "Deloise", "age": 44}
+```
+
+```
+..#                   >> 4
+..1                   >> {"name": "Alexa", "age": 34}
+..3                   >> {"name": "Deloise", "age": 44}
+..#.name              >> ["Gilbert","Alexa","May","Deloise"]
+..#(name="May").age   >> 57
+```
+
+The `ForEachLines` function will iterate through JSON lines.
+
+```go
+gjson.ForEachLine(json, func(line gjson.Result) bool{
+    println(line.String())
+    return true
+})
+```
+
+## Get nested array values
+
+Suppose you want all the last names from the following json:
+
+```json
+{
+  "programmers": [
+    {
+      "firstName": "Janet", 
+      "lastName": "McLaughlin", 
+    }, {
+      "firstName": "Elliotte", 
+      "lastName": "Hunter", 
+    }, {
+      "firstName": "Jason", 
+      "lastName": "Harold", 
+    }
+  ]
+}
+```
+
+You would use the path "programmers.#.lastName" like such:
+
+```go
+result := gjson.Get(json, "programmers.#.lastName")
+for _, name := range result.Array() {
+	println(name.String())
+}
+```
+
+You can also query an object inside an array:
+
+```go
+name := gjson.Get(json, `programmers.#(lastName="Hunter").firstName`)
+println(name.String())  // prints "Elliotte"
+```
+
+## Iterate through an object or array
+
+The `ForEach` function allows for quickly iterating through an object or array. 
+The key and value are passed to the iterator function for objects.
+Only the value is passed for arrays.
+Returning `false` from an iterator will stop iteration.
+
+```go
+result := gjson.Get(json, "programmers")
+result.ForEach(func(key, value gjson.Result) bool {
+	println(value.String()) 
+	return true // keep iterating
+})
+```
+
+## Simple Parse and Get
+
+There's a `Parse(json)` function that will do a simple parse, and `result.Get(path)` that will search a result.
+
+For example, all of these will return the same result:
+
+```go
+gjson.Parse(json).Get("name").Get("last")
+gjson.Get(json, "name").Get("last")
+gjson.Get(json, "name.last")
+```
+
+## Check for the existence of a value
+
+Sometimes you just want to know if a value exists. 
+
+```go
+value := gjson.Get(json, "name.last")
+if !value.Exists() {
+	println("no last name")
+} else {
+	println(value.String())
+}
+
+// Or as one step
+if gjson.Get(json, "name.last").Exists() {
+	println("has a last name")
+}
+```
+
+## Validate JSON
+
+The `Get*` and `Parse*` functions expects that the json is well-formed. Bad json will not panic, but it may return back unexpected results.
+
+If you are consuming JSON from an unpredictable source then you may want to validate prior to using GJSON.
+
+```go
+if !gjson.Valid(json) {
+	return errors.New("invalid json")
+}
+value := gjson.Get(json, "name.last")
+```
+
+## Unmarshal to a map
+
+To unmarshal to a `map[string]interface{}`:
+
+```go
+m, ok := gjson.Parse(json).Value().(map[string]interface{})
+if !ok {
+	// not a map
+}
+```
+
+## Working with Bytes
+
+If your JSON is contained in a `[]byte` slice, there's the [GetBytes](https://godoc.org/github.com/tidwall/gjson#GetBytes) function. This is preferred over `Get(string(data), path)`.
+
+```go
+var json []byte = ...
+result := gjson.GetBytes(json, path)
+```
+
+If you are using the `gjson.GetBytes(json, path)` function and you want to avoid converting `result.Raw` to a `[]byte`, then you can use this pattern:
+
+```go
+var json []byte = ...
+result := gjson.GetBytes(json, path)
+var raw []byte
+if result.Index > 0 {
+    raw = json[result.Index:result.Index+len(result.Raw)]
+} else {
+    raw = []byte(result.Raw)
+}
+```
+
+This is a best-effort no allocation sub slice of the original json. This method utilizes the `result.Index` field, which is the position of the raw data in the original json. It's possible that the value of `result.Index` equals zero, in which case the `result.Raw` is converted to a `[]byte`.
+
+## Get multiple values at once
+
+The `GetMany` function can be used to get multiple values at the same time.
+
+```go
+results := gjson.GetMany(json, "name.first", "name.last", "age")
+```
+
+The return value is a `[]Result`, which will always contain exactly the same number of items as the input paths.
+
+## Performance
+
+Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), 
+[ffjson](https://github.com/pquerna/ffjson), 
+[EasyJSON](https://github.com/mailru/easyjson),
+[jsonparser](https://github.com/buger/jsonparser),
+and [json-iterator](https://github.com/json-iterator/go)
+
+```
+BenchmarkGJSONGet-8                  3000000        372 ns/op          0 B/op         0 allocs/op
+BenchmarkGJSONUnmarshalMap-8          900000       4154 ns/op       1920 B/op        26 allocs/op
+BenchmarkJSONUnmarshalMap-8           600000       9019 ns/op       3048 B/op        69 allocs/op
+BenchmarkJSONDecoder-8                300000      14120 ns/op       4224 B/op       184 allocs/op
+BenchmarkFFJSONLexer-8               1500000       3111 ns/op        896 B/op         8 allocs/op
+BenchmarkEasyJSONLexer-8             3000000        887 ns/op        613 B/op         6 allocs/op
+BenchmarkJSONParserGet-8             3000000        499 ns/op         21 B/op         0 allocs/op
+BenchmarkJSONIterator-8              3000000        812 ns/op        544 B/op         9 allocs/op
+```
+
+JSON document used:
+
+```json
+{
+  "widget": {
+    "debug": "on",
+    "window": {
+      "title": "Sample Konfabulator Widget",
+      "name": "main_window",
+      "width": 500,
+      "height": 500
+    },
+    "image": { 
+      "src": "Images/Sun.png",
+      "hOffset": 250,
+      "vOffset": 250,
+      "alignment": "center"
+    },
+    "text": {
+      "data": "Click Here",
+      "size": 36,
+      "style": "bold",
+      "vOffset": 100,
+      "alignment": "center",
+      "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"
+    }
+  }
+}    
+```
+
+Each operation was rotated though one of the following search paths:
+
+```
+widget.window.name
+widget.image.hOffset
+widget.text.onMouseUp
+```
+
+*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).*
+
+
+## Contact
+Josh Baker [@tidwall](http://twitter.com/tidwall)
+
+## License
+
+GJSON source code is available under the MIT [License](/LICENSE).
--- a/vendor/github.com/tidwall/gjson/SYNTAX.md
+++ b/vendor/github.com/tidwall/gjson/SYNTAX.md
@ -0,0 +1,277 @@
+# GJSON Path Syntax
+
+A GJSON Path is a text string syntax that describes a search pattern for quickly retreiving values from a JSON payload.
+
+This document is designed to explain the structure of a GJSON Path through examples.
+
+- [Path structure](#path-structure)
+- [Basic](#basic)
+- [Wildcards](#wildcards)
+- [Escape Character](#escape-character)
+- [Arrays](#arrays)
+- [Queries](#queries)
+- [Dot vs Pipe](#dot-vs-pipe)
+- [Modifiers](#modifiers)
+- [Multipaths](#multipaths)
+
+The definitive implemenation is [github.com/tidwall/gjson](https://github.com/tidwall/gjson).  
+Use the [GJSON Playground](https://gjson.dev) to experiment with the syntax online.
+
+
+## Path structure
+
+A GJSON Path is intended to be easily expressed as a series of components seperated by a `.` character. 
+
+Along with `.` character, there are a few more that have special meaning, including `|`, `#`, `@`, `\`, `*`, and `?`.
+
+## Example
+
+Given this JSON
+
+```json
+{
+  "name": {"first": "Tom", "last": "Anderson"},
+  "age":37,
+  "children": ["Sara","Alex","Jack"],
+  "fav.movie": "Deer Hunter",
+  "friends": [
+    {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]},
+    {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]},
+    {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]}
+  ]
+}
+```
+
+The following GJSON Paths evaluate to the accompanying values.
+
+### Basic 
+
+In many cases you'll just want to retreive values by object name or array index.
+
+```go
+name.last              "Anderson"
+name.first             "Tom"
+age                    37
+children               ["Sara","Alex","Jack"]
+children.0             "Sara"
+children.1             "Alex"
+friends.1              {"first": "Roger", "last": "Craig", "age": 68}
+friends.1.first        "Roger"
+```
+
+### Wildcards
+
+A key may contain the special wildcard characters `*` and `?`. 
+The `*` will match on any zero+ characters, and `?` matches on any one character.
+
+```go
+child*.2               "Jack"
+c?ildren.0             "Sara"
+```
+
+### Escape character
+
+Special purpose characters, such as `.`, `*`, and `?` can be escaped with `\`. 
+
+```go
+fav\.movie             "Deer Hunter"
+```
+
+You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in source code.
+
+```go
+res := gjson.Get(json, "fav\\.movie")  // must escape the slash
+res := gjson.Get(json, `fav\.movie`)   // no need to escape the slash 
+
+```
+
+### Arrays
+
+The `#` character allows for digging into JSON Arrays.
+
+To get the length of an array you'll just use the `#` all by itself.
+
+```go
+friends.#              3
+friends.#.age         [44,68,47]
+```
+
+### Queries
+
+You can also query an array for the first match by  using `#(...)`, or find all matches with `#(...)#`. 
+Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` comparison operators, 
+and the simple pattern matching `%` (like) and `!%` (not like) operators.
+
+```go
+friends.#(last=="Murphy").first     "Dale"
+friends.#(last=="Murphy")#.first    ["Dale","Jane"]
+friends.#(age>45)#.last             ["Craig","Murphy"]
+friends.#(first%"D*").last          "Murphy"
+friends.#(first!%"D*").last         "Craig"
+```
+
+To query for a non-object value in an array, you can forgo the string to the right of the operator.
+
+```go
+children.#(!%"*a*")                 "Alex"
+children.#(%"*a*")#                 ["Sara","Jack"]
+```
+
+Nested queries are allowed.
+
+```go
+friends.#(nets.#(=="fb"))#.first  >> ["Dale","Roger"]
+```
+
+*Please note that prior to v1.3.0, queries used the `#[...]` brackets. This was
+changed in v1.3.0 as to avoid confusion with the new [multipath](#multipaths) 
+syntax. For backwards compatibility, `#[...]` will continue to work until the
+next major release.*
+
+### Dot vs Pipe
+
+The `.` is standard separator, but it's also possible to use a `|`. 
+In most cases they both end up returning the same results.
+The cases where`|` differs from `.` is when it's used after the `#` for [Arrays](#arrays) and [Queries](#queries). 
+
+Here are some examples
+
+```go
+friends.0.first                     "Dale"
+friends|0.first                     "Dale"
+friends.0|first                     "Dale"
+friends|0|first                     "Dale"
+friends|#                           3
+friends.#                           3
+friends.#(last="Murphy")#           [{"first": "Dale", "last": "Murphy", "age": 44},{"first": "Jane", "last": "Murphy", "age": 47}]
+friends.#(last="Murphy")#.first     ["Dale","Jane"]
+friends.#(last="Murphy")#|first     <non-existent>
+friends.#(last="Murphy")#.0         []
+friends.#(last="Murphy")#|0         {"first": "Dale", "last": "Murphy", "age": 44}
+friends.#(last="Murphy")#.#         []
+friends.#(last="Murphy")#|#         2
+```
+
+Let's break down a few of these.
+
+The path `friends.#(last="Murphy")#` all by itself results in
+
+```json
+[{"first": "Dale", "last": "Murphy", "age": 44},{"first": "Jane", "last": "Murphy", "age": 47}]
+```
+
+The `.first` suffix will process the `first` path on each array element *before* returning the results. Which becomes
+
+```json
+["Dale","Jane"]
+```
+
+But the `|first` suffix actually processes the `first` path *after* the previous result. 
+Since the previous result is an array, not an object, it's not possible to process 
+because `first` does not exist.
+
+Yet, `|0` suffix returns
+
+```json
+{"first": "Dale", "last": "Murphy", "age": 44}
+```
+
+Because `0` is the first index of the previous result.
+
+### Modifiers
+
+A modifier is a path component that performs custom processing on the JSON.
+
+For example, using the built-in `@reverse` modifier on the above JSON payload will reverse the `children` array:
+
+```go
+children.@reverse                   ["Jack","Alex","Sara"]
+children.@reverse.0                 "Jack"
+```
+
+There are currently the following built-in modifiers:
+
+- `@reverse`: Reverse an array or the members of an object.
+- `@ugly`: Remove all whitespace from JSON.
+- `@pretty`: Make the JSON more human readable.
+- `@this`: Returns the current element. It can be used to retrieve the root element.
+- `@valid`: Ensure the json document is valid.
+- `@flatten`: Flattens an array.
+- `@join`: Joins multiple objects into a single object.
+
+#### Modifier arguments
+
+A modifier may accept an optional argument. The argument can be a valid JSON payload or just characters.
+
+For example, the `@pretty` modifier takes a json object as its argument.
+
+```
+@pretty:{"sortKeys":true}
+```
+
+Which makes the json pretty and orders all of its keys.
+
+```json
+{
+  "age":37,
+  "children": ["Sara","Alex","Jack"],
+  "fav.movie": "Deer Hunter",
+  "friends": [
+    {"age": 44, "first": "Dale", "last": "Murphy"},
+    {"age": 68, "first": "Roger", "last": "Craig"},
+    {"age": 47, "first": "Jane", "last": "Murphy"}
+  ],
+  "name": {"first": "Tom", "last": "Anderson"}
+}
+```
+
+*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`. 
+Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.*
+
+#### Custom modifiers
+
+You can also add custom modifiers. 
+
+For example, here we create a modifier which makes the entire JSON payload upper or lower case.
+
+```go
+gjson.AddModifier("case", func(json, arg string) string {
+  if arg == "upper" {
+    return strings.ToUpper(json)
+  }
+  if arg == "lower" {
+    return strings.ToLower(json)
+  }
+  return json
+})
+"children.@case:upper"             ["SARA","ALEX","JACK"]
+"children.@case:lower.@reverse"    ["jack","alex","sara"]
+```
+
+### Multipaths
+
+Starting with v1.3.0, GJSON added the ability to join multiple paths together
+to form new documents. Wrapping comma-separated paths between `{...}` or 
+`[...]` will result in a new array or object, respectively.
+
+For example, using the given multipath 
+
+```
+{name.first,age,"the_murphys":friends.#(last="Murphy")#.first}
+```
+
+Here we selected the first name, age, and the first name for friends with the 
+last name "Murphy".
+
+You'll notice that an optional key can be provided, in this case 
+"the_murphys", to force assign a key to a value. Otherwise, the name of the 
+actual field will be used, in this case "first". If a name cannot be
+determined, then "_" is used.
+
+This results in
+
+```
+{"first":"Tom","age":37,"the_murphys":["Dale","Jane"]}
+```
+
+
--- a/vendor/github.com/tidwall/gjson/gjson.go
+++ b/vendor/github.com/tidwall/gjson/gjson.go
--- a/vendor/github.com/tidwall/gjson/go.mod
+++ b/vendor/github.com/tidwall/gjson/go.mod
@ -0,0 +1,8 @@
+module github.com/tidwall/gjson
+
+go 1.12
+
+require (
+	github.com/tidwall/match v1.0.1
+	github.com/tidwall/pretty v1.0.0
+)
--- a/vendor/github.com/tidwall/gjson/go.sum
+++ b/vendor/github.com/tidwall/gjson/go.sum
@ -0,0 +1,4 @@
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
--- a/vendor/github.com/tidwall/gjson/logo.png
+++ b/vendor/github.com/tidwall/gjson/logo.png
--- a/vendor/github.com/tidwall/match/.travis.yml
+++ b/vendor/github.com/tidwall/match/.travis.yml
@ -0,0 +1 @@
+language: go
--- a/vendor/github.com/tidwall/match/LICENSE
+++ b/vendor/github.com/tidwall/match/LICENSE
@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Josh Baker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--- a/vendor/github.com/tidwall/match/README.md
+++ b/vendor/github.com/tidwall/match/README.md
@ -0,0 +1,32 @@
+Match
+=====
+<a href="https://travis-ci.org/tidwall/match"><img src="https://img.shields.io/travis/tidwall/match.svg?style=flat-square" alt="Build Status"></a>
+<a href="https://godoc.org/github.com/tidwall/match"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
+
+Match is a very simple pattern matcher where '*' matches on any 
+number characters and '?' matches on any one character.
+
+Installing
+----------
+
+```
+go get -u github.com/tidwall/match
+```
+
+Example
+-------
+
+```go
+match.Match("hello", "*llo") 
+match.Match("jello", "?ello") 
+match.Match("hello", "h*o") 
+```
+
+
+Contact
+-------
+Josh Baker [@tidwall](http://twitter.com/tidwall)
+
+License
+-------
+Redcon source code is available under the MIT [License](/LICENSE).
--- a/vendor/github.com/tidwall/match/match.go
+++ b/vendor/github.com/tidwall/match/match.go
@ -0,0 +1,181 @@
+// Match provides a simple pattern matcher with unicode support.
+package match
+
+import "unicode/utf8"
+
+// Match returns true if str matches pattern. This is a very
+// simple wildcard match where '*' matches on any number characters
+// and '?' matches on any one character.
+
+// pattern:
+// 	{ term }
+// term:
+// 	'*'         matches any sequence of non-Separator characters
+// 	'?'         matches any single non-Separator character
+// 	c           matches character c (c != '*', '?', '\\')
+// 	'\\' c      matches character c
+//
+func Match(str, pattern string) bool {
+	if pattern == "*" {
+		return true
+	}
+	return deepMatch(str, pattern)
+}
+func deepMatch(str, pattern string) bool {
+	for len(pattern) > 0 {
+		if pattern[0] > 0x7f {
+			return deepMatchRune(str, pattern)
+		}
+		switch pattern[0] {
+		default:
+			if len(str) == 0 {
+				return false
+			}
+			if str[0] > 0x7f {
+				return deepMatchRune(str, pattern)
+			}
+			if str[0] != pattern[0] {
+				return false
+			}
+		case '?':
+			if len(str) == 0 {
+				return false
+			}
+		case '*':
+			return deepMatch(str, pattern[1:]) ||
+				(len(str) > 0 && deepMatch(str[1:], pattern))
+		}
+		str = str[1:]
+		pattern = pattern[1:]
+	}
+	return len(str) == 0 && len(pattern) == 0
+}
+
+func deepMatchRune(str, pattern string) bool {
+	var sr, pr rune
+	var srsz, prsz int
+
+	// read the first rune ahead of time
+	if len(str) > 0 {
+		if str[0] > 0x7f {
+			sr, srsz = utf8.DecodeRuneInString(str)
+		} else {
+			sr, srsz = rune(str[0]), 1
+		}
+	} else {
+		sr, srsz = utf8.RuneError, 0
+	}
+	if len(pattern) > 0 {
+		if pattern[0] > 0x7f {
+			pr, prsz = utf8.DecodeRuneInString(pattern)
+		} else {
+			pr, prsz = rune(pattern[0]), 1
+		}
+	} else {
+		pr, prsz = utf8.RuneError, 0
+	}
+	// done reading
+	for pr != utf8.RuneError {
+		switch pr {
+		default:
+			if srsz == utf8.RuneError {
+				return false
+			}
+			if sr != pr {
+				return false
+			}
+		case '?':
+			if srsz == utf8.RuneError {
+				return false
+			}
+		case '*':
+			return deepMatchRune(str, pattern[prsz:]) ||
+				(srsz > 0 && deepMatchRune(str[srsz:], pattern))
+		}
+		str = str[srsz:]
+		pattern = pattern[prsz:]
+		// read the next runes
+		if len(str) > 0 {
+			if str[0] > 0x7f {
+				sr, srsz = utf8.DecodeRuneInString(str)
+			} else {
+				sr, srsz = rune(str[0]), 1
+			}
+		} else {
+			sr, srsz = utf8.RuneError, 0
+		}
+		if len(pattern) > 0 {
+			if pattern[0] > 0x7f {
+				pr, prsz = utf8.DecodeRuneInString(pattern)
+			} else {
+				pr, prsz = rune(pattern[0]), 1
+			}
+		} else {
+			pr, prsz = utf8.RuneError, 0
+		}
+		// done reading
+	}
+
+	return srsz == 0 && prsz == 0
+}
+
+var maxRuneBytes = func() []byte {
+	b := make([]byte, 4)
+	if utf8.EncodeRune(b, '\U0010FFFF') != 4 {
+		panic("invalid rune encoding")
+	}
+	return b
+}()
+
+// Allowable parses the pattern and determines the minimum and maximum allowable
+// values that the pattern can represent.
+// When the max cannot be determined, 'true' will be returned
+// for infinite.
+func Allowable(pattern string) (min, max string) {
+	if pattern == "" || pattern[0] == '*' {
+		return "", ""
+	}
+
+	minb := make([]byte, 0, len(pattern))
+	maxb := make([]byte, 0, len(pattern))
+	var wild bool
+	for i := 0; i < len(pattern); i++ {
+		if pattern[i] == '*' {
+			wild = true
+			break
+		}
+		if pattern[i] == '?' {
+			minb = append(minb, 0)
+			maxb = append(maxb, maxRuneBytes...)
+		} else {
+			minb = append(minb, pattern[i])
+			maxb = append(maxb, pattern[i])
+		}
+	}
+	if wild {
+		r, n := utf8.DecodeLastRune(maxb)
+		if r != utf8.RuneError {
+			if r < utf8.MaxRune {
+				r++
+				if r > 0x7f {
+					b := make([]byte, 4)
+					nn := utf8.EncodeRune(b, r)
+					maxb = append(maxb[:len(maxb)-n], b[:nn]...)
+				} else {
+					maxb = append(maxb[:len(maxb)-n], byte(r))
+				}
+			}
+		}
+	}
+	return string(minb), string(maxb)
+}
+
+// IsPattern returns true if the string is a pattern.
+func IsPattern(str string) bool {
+	for i := 0; i < len(str); i++ {
+		if str[i] == '*' || str[i] == '?' {
+			return true
+		}
+	}
+	return false
+}
--- a/vendor/github.com/tidwall/pretty/.travis.yml
+++ b/vendor/github.com/tidwall/pretty/.travis.yml
@ -0,0 +1 @@
+language: go
--- a/vendor/github.com/tidwall/pretty/LICENSE
+++ b/vendor/github.com/tidwall/pretty/LICENSE
@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Josh Baker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--- a/vendor/github.com/tidwall/pretty/README.md
+++ b/vendor/github.com/tidwall/pretty/README.md
@ -0,0 +1,124 @@
+# Pretty
+[![Build Status](https://img.shields.io/travis/tidwall/pretty.svg?style=flat-square)](https://travis-ci.org/tidwall/prettty)
+[![Coverage Status](https://img.shields.io/badge/coverage-100%25-brightgreen.svg?style=flat-square)](http://gocover.io/github.com/tidwall/pretty)
+[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://godoc.org/github.com/tidwall/pretty) 
+
+
+Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads.
+
+Getting Started
+===============
+
+## Installing
+
+To start using Pretty, install Go and run `go get`:
+
+```sh
+$ go get -u github.com/tidwall/pretty
+```
+
+This will retrieve the library.
+
+## Pretty
+
+Using this example:
+
+```json
+{"name":  {"first":"Tom","last":"Anderson"},  "age":37,
+"children": ["Sara","Alex","Jack"],
+"fav.movie": "Deer Hunter", "friends": [
+    {"first": "Janet", "last": "Murphy", "age": 44}
+  ]}
+```
+
+The following code:
+```go
+result = pretty.Pretty(example)
+```
+
+Will format the json to:
+
+```json
+{
+  "name": {
+    "first": "Tom",
+    "last": "Anderson"
+  },
+  "age": 37,
+  "children": ["Sara", "Alex", "Jack"],
+  "fav.movie": "Deer Hunter",
+  "friends": [
+    {
+      "first": "Janet",
+      "last": "Murphy",
+      "age": 44
+    }
+  ]
+}
+```
+
+## Color
+
+Color will colorize the json for outputing to the screen. 
+
+```json
+result = pretty.Color(json, nil)
+```
+
+Will add color to the result for printing to the terminal.
+The second param is used for a customizing the style, and passing nil will use the default `pretty.TerminalStyle`.
+
+## Ugly
+
+The following code:
+```go
+result = pretty.Ugly(example)
+```
+
+Will format the json to:
+
+```json
+{"name":{"first":"Tom","last":"Anderson"},"age":37,"children":["Sara","Alex","Jack"],"fav.movie":"Deer Hunter","friends":[{"first":"Janet","last":"Murphy","age":44}]}```
+```
+
+
+## Customized output
+
+There's a `PrettyOptions(json, opts)` function which allows for customizing the output with the following options:
+
+```go
+type Options struct {
+	// Width is an max column width for single line arrays
+	// Default is 80
+	Width int
+	// Prefix is a prefix for all lines
+	// Default is an empty string
+	Prefix string
+	// Indent is the nested indentation
+	// Default is two spaces
+	Indent string
+	// SortKeys will sort the keys alphabetically
+	// Default is false
+	SortKeys bool
+}
+```
+## Performance
+
+Benchmarks of Pretty alongside the builtin `encoding/json` Indent/Compact methods.
+```
+BenchmarkPretty-8            1000000     1283 ns/op      720 B/op      2 allocs/op
+BenchmarkUgly-8              3000000      426 ns/op      240 B/op      1 allocs/op
+BenchmarkUglyInPlace-8       5000000      340 ns/op        0 B/op      0 allocs/op
+BenchmarkJSONIndent-8         300000     4628 ns/op     1069 B/op      4 allocs/op
+BenchmarkJSONCompact-8       1000000     2469 ns/op      758 B/op      4 allocs/op
+```
+
+*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.*
+
+## Contact
+Josh Baker [@tidwall](http://twitter.com/tidwall)
+
+## License
+
+Pretty source code is available under the MIT [License](/LICENSE).
+
--- a/vendor/github.com/tidwall/pretty/pretty.go
+++ b/vendor/github.com/tidwall/pretty/pretty.go
@ -0,0 +1,432 @@
+package pretty
+
+import (
+	"sort"
+)
+
+// Options is Pretty options
+type Options struct {
+	// Width is an max column width for single line arrays
+	// Default is 80
+	Width int
+	// Prefix is a prefix for all lines
+	// Default is an empty string
+	Prefix string
+	// Indent is the nested indentation
+	// Default is two spaces
+	Indent string
+	// SortKeys will sort the keys alphabetically
+	// Default is false
+	SortKeys bool
+}
+
+// DefaultOptions is the default options for pretty formats.
+var DefaultOptions = &Options{Width: 80, Prefix: "", Indent: "  ", SortKeys: false}
+
+// Pretty converts the input json into a more human readable format where each
+// element is on it's own line with clear indentation.
+func Pretty(json []byte) []byte { return PrettyOptions(json, nil) }
+
+// PrettyOptions is like Pretty but with customized options.
+func PrettyOptions(json []byte, opts *Options) []byte {
+	if opts == nil {
+		opts = DefaultOptions
+	}
+	buf := make([]byte, 0, len(json))
+	if len(opts.Prefix) != 0 {
+		buf = append(buf, opts.Prefix...)
+	}
+	buf, _, _, _ = appendPrettyAny(buf, json, 0, true,
+		opts.Width, opts.Prefix, opts.Indent, opts.SortKeys,
+		0, 0, -1)
+	if len(buf) > 0 {
+		buf = append(buf, '\n')
+	}
+	return buf
+}
+
+// Ugly removes insignificant space characters from the input json byte slice
+// and returns the compacted result.
+func Ugly(json []byte) []byte {
+	buf := make([]byte, 0, len(json))
+	return ugly(buf, json)
+}
+
+// UglyInPlace removes insignificant space characters from the input json
+// byte slice and returns the compacted result. This method reuses the
+// input json buffer to avoid allocations. Do not use the original bytes
+// slice upon return.
+func UglyInPlace(json []byte) []byte { return ugly(json, json) }
+
+func ugly(dst, src []byte) []byte {
+	dst = dst[:0]
+	for i := 0; i < len(src); i++ {
+		if src[i] > ' ' {
+			dst = append(dst, src[i])
+			if src[i] == '"' {
+				for i = i + 1; i < len(src); i++ {
+					dst = append(dst, src[i])
+					if src[i] == '"' {
+						j := i - 1
+						for ; ; j-- {
+							if src[j] != '\\' {
+								break
+							}
+						}
+						if (j-i)%2 != 0 {
+							break
+						}
+					}
+				}
+			}
+		}
+	}
+	return dst
+}
+
+func appendPrettyAny(buf, json []byte, i int, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) {
+	for ; i < len(json); i++ {
+		if json[i] <= ' ' {
+			continue
+		}
+		if json[i] == '"' {
+			return appendPrettyString(buf, json, i, nl)
+		}
+		if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' {
+			return appendPrettyNumber(buf, json, i, nl)
+		}
+		if json[i] == '{' {
+			return appendPrettyObject(buf, json, i, '{', '}', pretty, width, prefix, indent, sortkeys, tabs, nl, max)
+		}
+		if json[i] == '[' {
+			return appendPrettyObject(buf, json, i, '[', ']', pretty, width, prefix, indent, sortkeys, tabs, nl, max)
+		}
+		switch json[i] {
+		case 't':
+			return append(buf, 't', 'r', 'u', 'e'), i + 4, nl, true
+		case 'f':
+			return append(buf, 'f', 'a', 'l', 's', 'e'), i + 5, nl, true
+		case 'n':
+			return append(buf, 'n', 'u', 'l', 'l'), i + 4, nl, true
+		}
+	}
+	return buf, i, nl, true
+}
+
+type pair struct {
+	kstart, kend int
+	vstart, vend int
+}
+
+type byKey struct {
+	sorted bool
+	json   []byte
+	pairs  []pair
+}
+
+func (arr *byKey) Len() int {
+	return len(arr.pairs)
+}
+func (arr *byKey) Less(i, j int) bool {
+	key1 := arr.json[arr.pairs[i].kstart+1 : arr.pairs[i].kend-1]
+	key2 := arr.json[arr.pairs[j].kstart+1 : arr.pairs[j].kend-1]
+	return string(key1) < string(key2)
+}
+func (arr *byKey) Swap(i, j int) {
+	arr.pairs[i], arr.pairs[j] = arr.pairs[j], arr.pairs[i]
+	arr.sorted = true
+}
+
+func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) {
+	var ok bool
+	if width > 0 {
+		if pretty && open == '[' && max == -1 {
+			// here we try to create a single line array
+			max := width - (len(buf) - nl)
+			if max > 3 {
+				s1, s2 := len(buf), i
+				buf, i, _, ok = appendPrettyObject(buf, json, i, '[', ']', false, width, prefix, "", sortkeys, 0, 0, max)
+				if ok && len(buf)-s1 <= max {
+					return buf, i, nl, true
+				}
+				buf = buf[:s1]
+				i = s2
+			}
+		} else if max != -1 && open == '{' {
+			return buf, i, nl, false
+		}
+	}
+	buf = append(buf, open)
+	i++
+	var pairs []pair
+	if open == '{' && sortkeys {
+		pairs = make([]pair, 0, 8)
+	}
+	var n int
+	for ; i < len(json); i++ {
+		if json[i] <= ' ' {
+			continue
+		}
+		if json[i] == close {
+			if pretty {
+				if open == '{' && sortkeys {
+					buf = sortPairs(json, buf, pairs)
+				}
+				if n > 0 {
+					nl = len(buf)
+					buf = append(buf, '\n')
+				}
+				if buf[len(buf)-1] != open {
+					buf = appendTabs(buf, prefix, indent, tabs)
+				}
+			}
+			buf = append(buf, close)
+			return buf, i + 1, nl, open != '{'
+		}
+		if open == '[' || json[i] == '"' {
+			if n > 0 {
+				buf = append(buf, ',')
+				if width != -1 && open == '[' {
+					buf = append(buf, ' ')
+				}
+			}
+			var p pair
+			if pretty {
+				nl = len(buf)
+				buf = append(buf, '\n')
+				if open == '{' && sortkeys {
+					p.kstart = i
+					p.vstart = len(buf)
+				}
+				buf = appendTabs(buf, prefix, indent, tabs+1)
+			}
+			if open == '{' {
+				buf, i, nl, _ = appendPrettyString(buf, json, i, nl)
+				if sortkeys {
+					p.kend = i
+				}
+				buf = append(buf, ':')
+				if pretty {
+					buf = append(buf, ' ')
+				}
+			}
+			buf, i, nl, ok = appendPrettyAny(buf, json, i, pretty, width, prefix, indent, sortkeys, tabs+1, nl, max)
+			if max != -1 && !ok {
+				return buf, i, nl, false
+			}
+			if pretty && open == '{' && sortkeys {
+				p.vend = len(buf)
+				if p.kstart > p.kend || p.vstart > p.vend {
+					// bad data. disable sorting
+					sortkeys = false
+				} else {
+					pairs = append(pairs, p)
+				}
+			}
+			i--
+			n++
+		}
+	}
+	return buf, i, nl, open != '{'
+}
+func sortPairs(json, buf []byte, pairs []pair) []byte {
+	if len(pairs) == 0 {
+		return buf
+	}
+	vstart := pairs[0].vstart
+	vend := pairs[len(pairs)-1].vend
+	arr := byKey{false, json, pairs}
+	sort.Sort(&arr)
+	if !arr.sorted {
+		return buf
+	}
+	nbuf := make([]byte, 0, vend-vstart)
+	for i, p := range pairs {
+		nbuf = append(nbuf, buf[p.vstart:p.vend]...)
+		if i < len(pairs)-1 {
+			nbuf = append(nbuf, ',')
+			nbuf = append(nbuf, '\n')
+		}
+	}
+	return append(buf[:vstart], nbuf...)
+}
+
+func appendPrettyString(buf, json []byte, i, nl int) ([]byte, int, int, bool) {
+	s := i
+	i++
+	for ; i < len(json); i++ {
+		if json[i] == '"' {
+			var sc int
+			for j := i - 1; j > s; j-- {
+				if json[j] == '\\' {
+					sc++
+				} else {
+					break
+				}
+			}
+			if sc%2 == 1 {
+				continue
+			}
+			i++
+			break
+		}
+	}
+	return append(buf, json[s:i]...), i, nl, true
+}
+
+func appendPrettyNumber(buf, json []byte, i, nl int) ([]byte, int, int, bool) {
+	s := i
+	i++
+	for ; i < len(json); i++ {
+		if json[i] <= ' ' || json[i] == ',' || json[i] == ':' || json[i] == ']' || json[i] == '}' {
+			break
+		}
+	}
+	return append(buf, json[s:i]...), i, nl, true
+}
+
+func appendTabs(buf []byte, prefix, indent string, tabs int) []byte {
+	if len(prefix) != 0 {
+		buf = append(buf, prefix...)
+	}
+	if len(indent) == 2 && indent[0] == ' ' && indent[1] == ' ' {
+		for i := 0; i < tabs; i++ {
+			buf = append(buf, ' ', ' ')
+		}
+	} else {
+		for i := 0; i < tabs; i++ {
+			buf = append(buf, indent...)
+		}
+	}
+	return buf
+}
+
+// Style is the color style
+type Style struct {
+	Key, String, Number [2]string
+	True, False, Null   [2]string
+	Append              func(dst []byte, c byte) []byte
+}
+
+func hexp(p byte) byte {
+	switch {
+	case p < 10:
+		return p + '0'
+	default:
+		return (p - 10) + 'a'
+	}
+}
+
+// TerminalStyle is for terminals
+var TerminalStyle = &Style{
+	Key:    [2]string{"\x1B[94m", "\x1B[0m"},
+	String: [2]string{"\x1B[92m", "\x1B[0m"},
+	Number: [2]string{"\x1B[93m", "\x1B[0m"},
+	True:   [2]string{"\x1B[96m", "\x1B[0m"},
+	False:  [2]string{"\x1B[96m", "\x1B[0m"},
+	Null:   [2]string{"\x1B[91m", "\x1B[0m"},
+	Append: func(dst []byte, c byte) []byte {
+		if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') {
+			dst = append(dst, "\\u00"...)
+			dst = append(dst, hexp((c>>4)&0xF))
+			return append(dst, hexp((c)&0xF))
+		}
+		return append(dst, c)
+	},
+}
+
+// Color will colorize the json. The style parma is used for customizing
+// the colors. Passing nil to the style param will use the default
+// TerminalStyle.
+func Color(src []byte, style *Style) []byte {
+	if style == nil {
+		style = TerminalStyle
+	}
+	apnd := style.Append
+	if apnd == nil {
+		apnd = func(dst []byte, c byte) []byte {
+			return append(dst, c)
+		}
+	}
+	type stackt struct {
+		kind byte
+		key  bool
+	}
+	var dst []byte
+	var stack []stackt
+	for i := 0; i < len(src); i++ {
+		if src[i] == '"' {
+			key := len(stack) > 0 && stack[len(stack)-1].key
+			if key {
+				dst = append(dst, style.Key[0]...)
+			} else {
+				dst = append(dst, style.String[0]...)
+			}
+			dst = apnd(dst, '"')
+			for i = i + 1; i < len(src); i++ {
+				dst = apnd(dst, src[i])
+				if src[i] == '"' {
+					j := i - 1
+					for ; ; j-- {
+						if src[j] != '\\' {
+							break
+						}
+					}
+					if (j-i)%2 != 0 {
+						break
+					}
+				}
+			}
+			if key {
+				dst = append(dst, style.Key[1]...)
+			} else {
+				dst = append(dst, style.String[1]...)
+			}
+		} else if src[i] == '{' || src[i] == '[' {
+			stack = append(stack, stackt{src[i], src[i] == '{'})
+			dst = apnd(dst, src[i])
+		} else if (src[i] == '}' || src[i] == ']') && len(stack) > 0 {
+			stack = stack[:len(stack)-1]
+			dst = apnd(dst, src[i])
+		} else if (src[i] == ':' || src[i] == ',') && len(stack) > 0 && stack[len(stack)-1].kind == '{' {
+			stack[len(stack)-1].key = !stack[len(stack)-1].key
+			dst = apnd(dst, src[i])
+		} else {
+			var kind byte
+			if (src[i] >= '0' && src[i] <= '9') || src[i] == '-' {
+				kind = '0'
+				dst = append(dst, style.Number[0]...)
+			} else if src[i] == 't' {
+				kind = 't'
+				dst = append(dst, style.True[0]...)
+			} else if src[i] == 'f' {
+				kind = 'f'
+				dst = append(dst, style.False[0]...)
+			} else if src[i] == 'n' {
+				kind = 'n'
+				dst = append(dst, style.Null[0]...)
+			} else {
+				dst = apnd(dst, src[i])
+			}
+			if kind != 0 {
+				for ; i < len(src); i++ {
+					if src[i] <= ' ' || src[i] == ',' || src[i] == ':' || src[i] == ']' || src[i] == '}' {
+						i--
+						break
+					}
+					dst = apnd(dst, src[i])
+				}
+				if kind == '0' {
+					dst = append(dst, style.Number[1]...)
+				} else if kind == 't' {
+					dst = append(dst, style.True[1]...)
+				} else if kind == 'f' {
+					dst = append(dst, style.False[1]...)
+				} else if kind == 'n' {
+					dst = append(dst, style.Null[1]...)
+				}
+			}
+		}
+	}
+	return dst
+}
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@ -235,6 +235,12 @@ github.com/spf13/pflag
 github.com/spf13/viper
 # github.com/stretchr/testify v1.5.1
 github.com/stretchr/testify/assert
+# github.com/tidwall/gjson v1.6.0
+github.com/tidwall/gjson
+# github.com/tidwall/match v1.0.1
+github.com/tidwall/match
+# github.com/tidwall/pretty v1.0.0
+github.com/tidwall/pretty
 # github.com/urfave/cli v1.20.0
 github.com/urfave/cli
 # github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e