Merge pull request #2011 from stashapp/develop

Merge to master for v0.11.0 release

.github/workflows/build.yml

@@ -47,15 +47,18 @@ jobs:
     - name: Cache go build
       uses: actions/cache@v2
       env:
-        cache-name: cache-go-cache
+        # increment the number suffix to bump the cache
+        cache-name: cache-go-cache-1
       with:
         path: .go-cache
-        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }}
+        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('go.mod', '**/go.sum') }}
 
     - name: Start build container
+      env:
+        official-build: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/develop') || (github.event_name == 'release' && github.ref != 'refs/tags/latest_develop') }}
       run: |
         mkdir -p .go-cache
-        docker run -d --name build --mount type=bind,source="$(pwd)",target=/stash,consistency=delegated --mount type=bind,source="$(pwd)/.go-cache",target=/root/.cache/go-build,consistency=delegated -w /stash $COMPILER_IMAGE tail -f /dev/null
+        docker run -d --name build --mount type=bind,source="$(pwd)",target=/stash,consistency=delegated --mount type=bind,source="$(pwd)/.go-cache",target=/root/.cache/go-build,consistency=delegated --env OFFICIAL_BUILD=${{ env.official-build }} -w /stash $COMPILER_IMAGE tail -f /dev/null
 
     - name: Pre-install
       run: docker exec -t build /bin/bash -c "make pre-ui"

.golangci.yml

@@ -17,30 +17,35 @@ linters:
     - typecheck
     - unused
     - varcheck
-    # Linters added by the stash project
-    # - bodyclose
+    # Linters added by the stash project.
     - dogsled
-    # - errorlint
+    - errorlint
     # - exhaustive
     - exportloopref
-    # - goconst
-    # - gocritic
+    - gocritic
     # - goerr113
     - gofmt
-    # - gosec
+    # - gomnd
     # - ifshort
     - misspell
     # - nakedret
-    # - noctx
-    # - paralleltest
+    - noctx
     - revive
     - rowserrcheck
     - sqlclosecheck
 
+# Project-specific linter overrides
 linters-settings:
   gofmt:
     simplify: false
 
+  errorlint:
+    # Disable errorf because there are false positives, where you don't want to wrap
+    #  an error.
+    errorf: false
+    asserts: true
+    comparison: true
+
   revive:
     ignore-generated-header: true
     severity: error
@@ -79,4 +84,8 @@ linters-settings:
       - name: unused-parameter
         disabled: true
       - name: unreachable-code
-      - name: redefines-builtin-id
+      - name: redefines-builtin-id
+
+  rowserrcheck:
+    packages:
+      - github.com/jmoiron/sqlx

.goreleaser.yml

@@ -1,76 +0,0 @@
-project_name: stash
-before:
-  hooks:
-    - go mod download
-builds:
-  - binary: stash-win
-    ldflags:
-      - "-extldflags '-static'"
-    env:
-      - CGO_ENABLED=1
-      - CC=x86_64-w64-mingw32-gcc
-      - CXX=x86_64-w64-mingw32-g++
-    flags:
-      - -tags
-      - extended
-    goos:
-      - windows
-    goarch:
-      - amd64
-  - binary: stash-osx
-    env:
-      - CGO_ENABLED=1
-      - CC=o64-clang
-      - CXX=o64-clang++
-    flags:
-      - -tags
-      - extended
-    goos:
-      - darwin
-    goarch:
-      - amd64
-  - binary: stash-osx-applesilicon
-    env:
-      - CGO_ENABLED=1
-      - CC=oa64-clang
-      - CXX=oa64-clang++
-    flags:
-      - -tags
-      - extended
-    goos:
-      - darwin
-    goarch:
-      - arm64
-  - binary: stash-linux
-    env:
-      - CGO_ENABLED=1
-    flags:
-      - -tags
-      - extended
-    goos:
-      - linux
-    goarch:
-      - amd64
-archive:
-  format: tar.gz
-  format_overrides:
-    - goos: windows
-      format: zip
-  name_template: "{{.ProjectName}}_{{.Version}}_{{.Os}}-{{.Arch}}"
-  replacements:
-    amd64: 64bit
-    386: 32bit
-    arm: ARM
-    arm64: ARM64
-    darwin: macOS
-    linux: Linux
-    windows: Windows
-    openbsd: OpenBSD
-    netbsd: NetBSD
-    freebsd: FreeBSD
-    dragonfly: DragonFlyBSD
-  files:
-    - README.md
-    - LICENSE
-release:
-  draft: true

.travis.yml.disabled

@@ -1,117 +0,0 @@
-if: tag != latest_develop # dont build for the latest_develop tagged version
-
-dist: xenial
-git:
-  depth: false
-language: go
-go:
-- 1.17.x
-services:
-- docker
-before_install:
-  - set -e
-  # Configure environment so changes are picked up when the Docker daemon is restarted after upgrading
-  - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
-  - export DOCKER_CLI_EXPERIMENTAL=enabled
-  # Upgrade to Docker CE 19.03 for BuildKit support
-  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-  - sudo apt-get update
-  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-  # install binfmt docker container, this container uses qemu to run arm programs transparently allowng docker to build arm 6,7,8 containers.
-  - docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64
-  # Show info to simplify debugging and create a builder that can build the platforms we need
-  - docker info
-  - docker buildx create --name builder --use
-  - docker buildx inspect --bootstrap
-  - docker buildx ls
-
-install:
-- echo -e "machine github.com\n  login $CI_USER_TOKEN" > ~/.netrc
-- nvm install 12
-- travis_retry make pre-ui
-- make generate
-- CI=false make ui-validate ui-only
-#- go get -v github.com/mgechev/revive
-script:
-# left lint off to avoid getting extra dependency
-#- make lint
-- make fmt-check vet it
-after_success:
-- docker pull stashapp/compiler:5
-- sh ./scripts/cross-compile.sh
-- git describe --tags --exclude latest_develop | tee CHECKSUMS_SHA1
-- sha1sum dist/stash-* | sed 's/dist\///g' | tee -a CHECKSUMS_SHA1
-- 'if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then sh ./scripts/upload-pull-request.sh; fi'
-before_deploy:
-# push the latest tag when on the develop branch
-- if [ "$TRAVIS_BRANCH" = "develop" ]; then git tag -f latest_develop; git push -f --tags; fi
-- export RELEASE_DATE=$(date +'%Y-%m-%d %H:%M:%S %Z')
-- export STASH_VERSION=$(git describe --tags --exclude latest_develop)
-# set TRAVIS_TAG explcitly to the version so that it doesn't pick up latest_develop
-- if [ "$TRAVIS_BRANCH" = "master" ]; then export TRAVIS_TAG=${STASH_VERSION}; fi
-deploy:
-  # latest develop release
-  - provider: releases
-    # use the v2 release provider for proper release note setting
-    edge: true
-    api_key:
-      secure: tGJ2q62CfPdayid2qEtW2aGRhMgCl3lBXYYQqp3eH0vFgIIf6cs7IDX7YC/x3XKMEQ/iMLZmtCXZvSTqNrD6Sk7MSnt30GIs+4uxIZDnnd8mV5X3K4n4gjD+NAORc4DrQBvUGrYMKJsR5gtkH0nu6diWb1o1If7OiJEuCPRhrmQYcza7NUdABnA9Z2wn2RNUV9Ga33WUCqLMEU5GtNBlfQPiP/khCQrqn/ocR6wUjYut3J6YagzqH4wsfJi3glHyWtowcNIw1LZi5zFxHD/bRBT4Tln7yypkjWNq9eQILA6i6kRUGf7ggyTx26/k8n4tnu+QD0vVh4EcjlThpU/LGyUXzKrrxjRwaDZnM0oYxg5AfHcBuAiAdo0eWnV3lEWRfTJMIVb9MPf4qDmzR4RREfB5OXOxwq3ODeCcJE8sTIMD/wBPZrlqS/QrRpND2gn2X4snkVukN9t9F4CMTFMtVSzFV7TDJW5E5Lq6VEExulteQhs6kcK9NRPNAaLgRQAw7X9kVWfDtiGUP+fE2i8F9Bo8bm7sOT5O5VPMPykx3EgeNg1IqIgMTCsMlhMJT4xBJoQUgmd2wWyf3Ryw+P+sFgdb5Sd7+lFgJBjMUUoOxMxAOiEgdFvCXcr+/Udyz2RdtetU1/6VzXzLPcKOw0wubZeBkISqu7o9gpfdMP9Eq00=
-    file:
-      - dist/stash-osx
-      - dist/stash-osx-applesilicon
-      - dist/stash-win.exe
-      - dist/stash-linux
-      - dist/stash-linux-arm64v8
-      - dist/stash-linux-arm32v7
-      - dist/stash-pi
-      - CHECKSUMS_SHA1
-    skip_cleanup: true
-    overwrite: true
-    name: "${STASH_VERSION}: Latest development build"
-    release_notes: "**${RELEASE_DATE}**\n This is always the latest committed version on the develop branch. Use as your own risk!"
-    prerelease: true
-    on:
-      repo: stashapp/stash
-      branch: develop
-  # docker image build for develop release
-  - provider: script
-    skip_cleanup: true
-    script: bash ./docker/ci/x86_64/docker_push.sh development
-    on:
-      repo: stashapp/stash
-      branch: develop
-  # official master release - only build when tagged
-  - provider: releases
-    api_key:
-      secure: tGJ2q62CfPdayid2qEtW2aGRhMgCl3lBXYYQqp3eH0vFgIIf6cs7IDX7YC/x3XKMEQ/iMLZmtCXZvSTqNrD6Sk7MSnt30GIs+4uxIZDnnd8mV5X3K4n4gjD+NAORc4DrQBvUGrYMKJsR5gtkH0nu6diWb1o1If7OiJEuCPRhrmQYcza7NUdABnA9Z2wn2RNUV9Ga33WUCqLMEU5GtNBlfQPiP/khCQrqn/ocR6wUjYut3J6YagzqH4wsfJi3glHyWtowcNIw1LZi5zFxHD/bRBT4Tln7yypkjWNq9eQILA6i6kRUGf7ggyTx26/k8n4tnu+QD0vVh4EcjlThpU/LGyUXzKrrxjRwaDZnM0oYxg5AfHcBuAiAdo0eWnV3lEWRfTJMIVb9MPf4qDmzR4RREfB5OXOxwq3ODeCcJE8sTIMD/wBPZrlqS/QrRpND2gn2X4snkVukN9t9F4CMTFMtVSzFV7TDJW5E5Lq6VEExulteQhs6kcK9NRPNAaLgRQAw7X9kVWfDtiGUP+fE2i8F9Bo8bm7sOT5O5VPMPykx3EgeNg1IqIgMTCsMlhMJT4xBJoQUgmd2wWyf3Ryw+P+sFgdb5Sd7+lFgJBjMUUoOxMxAOiEgdFvCXcr+/Udyz2RdtetU1/6VzXzLPcKOw0wubZeBkISqu7o9gpfdMP9Eq00=
-    file:
-      - dist/stash-osx
-      - dist/stash-osx-applesilicon
-      - dist/stash-win.exe
-      - dist/stash-linux
-      - dist/stash-linux-arm64v8
-      - dist/stash-linux-arm32v7
-      - dist/stash-pi
-      - CHECKSUMS_SHA1
-    # make the release a draft so the maintainers can confirm before releasing
-    draft: true
-    skip_cleanup: true
-    overwrite: true
-    # don't write the body. To be done manually for now. In future we might
-    # want to generate the changelog or get it from a file
-    name: ${STASH_VERSION}
-    on:
-      repo: stashapp/stash
-      tags: true
-      # make sure we don't release using the latest_develop tag
-      condition: $TRAVIS_TAG != latest_develop
-  # docker image build for master release
-  - provider: script
-    skip_cleanup: true
-    script: bash ./docker/ci/x86_64/docker_push.sh latest
-    on:
-      repo: stashapp/stash
-      tags: true
-      # make sure we don't release using the latest_develop tag
-      condition: $TRAVIS_TAG != latest_develop

Makefile

@@ -41,8 +41,13 @@ ifndef STASH_VERSION
 	$(eval STASH_VERSION := $(shell git describe --tags --exclude latest_develop))
 endif
 
+ifndef OFFICIAL_BUILD
+    $(eval OFFICIAL_BUILD := false)
+endif
+
 build: pre-build
 	$(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/pkg/api.version=$(STASH_VERSION)' -X 'github.com/stashapp/stash/pkg/api.buildstamp=$(BUILD_DATE)' -X 'github.com/stashapp/stash/pkg/api.githash=$(GITHASH)')
+	$(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/pkg/api.officialBuild=$(OFFICIAL_BUILD)')
 	go build $(OUTPUT) -mod=vendor -v -tags "sqlite_omit_load_extension osusergo netgo" $(GO_BUILD_FLAGS) -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS)"
 
 # strips debug symbols from the release build
@@ -195,5 +200,5 @@ validate-backend: lint it
 
 # locally builds and tags a 'stash/build' docker image
 .PHONY: docker-build
-docker-build:
-	docker build -t stash/build -f docker/build/x86_64/Dockerfile .
+docker-build: pre-build
+	docker build --build-arg GITHASH=$(GITHASH) --build-arg STASH_VERSION=$(STASH_VERSION) -t stash/build -f docker/build/x86_64/Dockerfile .

README.md

@@ -1,83 +1,46 @@
 # Stash
+https://stashapp.cc
 
+[![Build](https://github.com/stashapp/stash/actions/workflows/build.yml/badge.svg?branch=develop&event=push)](https://github.com/stashapp/stash/actions/workflows/build.yml)
+[![Docker pulls](https://img.shields.io/docker/pulls/stashapp/stash.svg)](https://hub.docker.com/r/stashapp/Stash 'DockerHub')
 [![Go Report Card](https://goreportcard.com/badge/github.com/stashapp/stash)](https://goreportcard.com/report/github.com/stashapp/stash)
 [![Discord](https://img.shields.io/discord/559159668438728723.svg?logo=discord)](https://discord.gg/2TsNFKt)
 
-https://stashapp.cc
+### **Stash is a self-hosted webapp written in Go which organizes and serves your porn.**
+![demo image](docs/readme_assets/demo_image.png)
 
-**Stash is a locally hosted web-based app written in Go which organizes and serves your porn.**
-
-* It can gather information about videos in your collection from the internet, and is extensible through the use of community-built plugins for a large number of content producers.
-* It supports a wide variety of both video and image formats.
+* Stash gathers information about videos in your collection from the internet, and is extensible through the use of community-built plugins for a large number of content producers and sites.
+* Stash supports a wide variety of both video and image formats.
 * You can tag videos and find them later.
-* It provides statistics about performers, tags, studios and other things.
+* Stash provides statistics about performers, tags, studios and more.
 
 You can [watch a SFW demo video](https://vimeo.com/545323354) to see it in action.
 
 For further information you can [read the in-app manual](ui/v2.5/src/docs/en).
 
-# Installing stash
+# Installing Stash
 
-## via Docker
+<img src="docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="docs/readme_assets/mac_logo.svg" width="100%" height="75"> MacOS| <img src="docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="docs/readme_assets/docker_logo.svg" width="100%" height="75"> Docker
+:---:|:---:|:---:|:---:
+[Latest Release](https://github.com/stashapp/stash/releases/latest/download/stash-win.exe) <br /> <sup><sub>[Development Preview](https://github.com/stashapp/stash/releases/download/latest_develop/stash-win.exe)</sub></sup> | [Latest Release (Apple Silicon)](https://github.com/stashapp/stash/releases/latest/download/stash-osx-applesilicon) <br /> <sup><sub>[Development Preview (Apple Silicon)](https://github.com/stashapp/stash/releases/download/latest_develop/stash-osx-applesilicon)</sub></sup> <br>[Latest Release (Intel)](https://github.com/stashapp/stash/releases/latest/download/stash-osx) <br /> <sup><sub>[Development Preview (Intel)](https://github.com/stashapp/stash/releases/download/latest_develop/stash-osx)</sub></sup> | [Latest Release (amd64)](https://github.com/stashapp/stash/releases/latest/download/stash-linux) <br /> <sup><sub>[Development Preview (amd64)](https://github.com/stashapp/stash/releases/download/latest_develop/stash-linux)</sub></sup> <br /> [More Architectures...](https://github.com/stashapp/stash/releases/latest) | [Instructions](docker/production/README.md) <br /> <sup><sub> [Sample docker-compose.yml](docker/production/docker-compose.yml)</sub></sup>
 
-Follow [this README.md in the docker directory.](docker/production/README.md)
-
-## Pre-Compiled Binaries
-
-The Stash server runs on macOS, Windows, and Linux.  Download the [latest release here](https://github.com/stashapp/stash/releases).
-
-Run the executable (double click the exe on windows or run `./stash-osx` / `./stash-linux` from the terminal on macOS / Linux) and navigate to either https://localhost:9999 or http://localhost:9999 to get started.
+## Getting Started
+Run the executable (double click the exe on windows or run `./stash-osx` / `./stash-linux` from the terminal on macOS / Linux) to get started.
 
 *Note for Windows users:* Running the app might present a security prompt since the binary isn't yet signed.  Bypass this by clicking "more info" and then the "run anyway" button.
 
 #### FFMPEG
-
-If stash is unable to find or download FFMPEG then download it yourself from the link for your platform:
-
-* [macOS ffmpeg](https://evermeet.cx/ffmpeg/ffmpeg-4.3.1.zip), [macOS ffprobe](https://evermeet.cx/ffmpeg/ffprobe-4.3.1.zip)
-* [Windows](https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.zip)
-* [Linux](https://www.johnvansickle.com/ffmpeg/)
-
-The `ffmpeg(.exe)` and `ffprobe(.exe)` files should be placed in `~/.stash` on macOS / Linux or `C:\Users\YourUsername\.stash` on Windows.
+Stash requires ffmpeg. If you don't have it installed, Stash will download a copy for you. It is recommended that Linux users install `ffmpeg` from their distro's package manager.
 
 # Usage
 
 ## Quickstart Guide
-1) Download and install Stash and its dependencies
-2) Run Stash. It will prompt you for some configuration options and a directory to index (you can also do this step afterward)
-3) After configuration, launch your web browser and navigate to the URL shown within the Stash app.
+Download and run Stash. It will prompt you for some configuration options and a directory to index (you can also do this step afterward)
 
-**Note that Stash does not currently retrieve and organize information about your entire library automatically.**  You will need to help it along through the use of [scrapers](blob/develop/ui/v2.5/src/docs/en/Scraping.md).  The Stash community has developed scrapers for many popular data sources which can be downloaded and installed from [this repository](https://github.com/stashapp/CommunityScrapers).
+**If you'd like to automatically retrieve and organize information about your entire library,**  You will need to download some [scrapers](https://github.com/stashapp/stash/tree/develop/ui/v2.5/src/docs/en/Scraping.md). The Stash community has developed scrapers for many popular data sources which can be downloaded and installed from [this repository](https://github.com/stashapp/CommunityScrapers).
 
 The simplest way to tag a large number of files is by using the [Tagger](https://github.com/stashapp/stash/blob/develop/ui/v2.5/src/docs/en/Tagger.md) which uses filename keywords to help identify the file and pull in scene and performer information from our stash-box database. Note that this data source is not comprehensive and you may need to use the scrapers to identify some of your media.
 
-## CLI
-
-Stash runs as a command-line app and local web server.  There are some command-line options available, which you can see by running `stash --help`.
-
-For example, to run stash locally on port 80 run it like this (OSX / Linux) `stash --host 127.0.0.1 --port 80`
-
-## SSL (HTTPS)
-
-Stash can run over HTTPS with some additional work.  First you must generate a SSL certificate and key combo.  Here is an example using openssl:
-
-`openssl req -x509 -newkey rsa:4096 -sha256 -days 7300 -nodes -keyout stash.key -out stash.crt -extensions san -config <(echo "[req]"; echo distinguished_name=req; echo "[san]"; echo subjectAltName=DNS:stash.server,IP:127.0.0.1) -subj /CN=stash.server`
-
-This command would need customizing for your environment.  [This link](https://stackoverflow.com/questions/10175812/how-to-create-a-self-signed-certificate-with-openssl) might be useful.
-
-Once you have a certificate and key file name them `stash.crt` and `stash.key` and place them in the same directory as the `config.yml` file, or the `~/.stash` directory.  Stash detects these and starts up using HTTPS rather than HTTP.
-
-## Basepath rewriting
-
-The basepath defaults to `/`. When running stash via a reverse proxy in a subpath, the basepath can be changed by having the reverse proxy pass `X-Forwarded-Prefix` (and optionally `X-Forwarded-Port`) headers. When detects these headers, it alters the basepath URL of the UI.
-
-# Customization
-
-## Themes and CSS Customization
-There is a [directory of community-created themes](https://github.com/stashapp/stash/wiki/Themes) on our Wiki, along with instructions on how to install them.
-
-You can also make Stash interface fit your desired style with [Custom CSS snippets](https://github.com/stashapp/stash/wiki/Custom-CSS-snippets) and [CSS Tweaks](https://github.com/stashapp/stash/wiki/CSS-Tweaks).
-
 # Support (FAQ)
 
 Answers to other Frequently Asked Questions can be found [on our Wiki](https://github.com/stashapp/stash/wiki/FAQ)
@@ -85,73 +48,18 @@ Answers to other Frequently Asked Questions can be found [on our Wiki](https://g
 For issues not addressed there, there are a few options.
 
 * Read the [Wiki](https://github.com/stashapp/stash/wiki)
-* Check the in-app documentation (also available [here](https://github.com/stashapp/stash/tree/develop/ui/v2.5/src/docs/en)
+* Check the in-app documentation, in the top right corner of the app (also available [here](https://github.com/stashapp/stash/tree/develop/ui/v2.5/src/docs/en)
 * Join the [Discord server](https://discord.gg/2TsNFKt), where the community can offer support.
 
-# Compiling From Source Code
+# Customization
 
-## Pre-requisites
+## Themes and CSS Customization
+There is a [directory of community-created themes](https://github.com/stashapp/stash/wiki/Themes) on our Wiki, along with instructions on how to install them.
 
-* [Go](https://golang.org/dl/)
-* [GolangCI](https://golangci-lint.run/) - A meta-linter which runs several linters in parallel
-  * To install, follow the [local installation instructions](https://golangci-lint.run/usage/install/#local-installation)
-* [Yarn](https://yarnpkg.com/en/docs/install) - Yarn package manager
-  * Run `yarn install --frozen-lockfile` in the `stash/ui/v2.5` folder (before running make generate for first time).
+You can also make Stash interface fit your desired style with [Custom CSS snippets](https://github.com/stashapp/stash/wiki/Custom-CSS-snippets).
 
-NOTE: You may need to run the `go get` commands outside the project directory to avoid modifying the projects module file.
+# For Developers
 
-## Environment
+Pull requests are welcome! 
 
-### macOS
-
-TODO
-
-### Windows
-
-1. Download and install [Go for Windows](https://golang.org/dl/)
-2. Download and install [MingW](https://sourceforge.net/projects/mingw-w64/)
-3. Search for "advanced system settings" and open the system properties dialog.
-    1. Click the `Environment Variables` button
-    2. Under system variables find the `Path`.  Edit and add `C:\Program Files\mingw-w64\*\mingw64\bin` (replace * with the correct path).
-
-NOTE: The `make` command in Windows will be `mingw32-make` with MingW.
-
-## Commands
-
-* `make generate` - Generate Go and UI GraphQL files
-* `make build` - Builds the binary (make sure to build the UI as well... see below)
-* `make docker-build` - Locally builds and tags a complete 'stash/build' docker image
-* `make pre-ui` - Installs the UI dependencies. Only needs to be run once before building the UI for the first time, or if the dependencies are updated
-* `make fmt-ui` - Formats the UI source code
-* `make ui` - Builds the frontend
-* `make lint` - Run the linter on the backend
-* `make fmt` - Run `go fmt`
-* `make it` - Run the unit and integration tests
-* `make validate` - Run all of the tests and checks required to submit a PR
-* `make ui-start` - Runs the UI in development mode. Requires a running stash server to connect to. Stash port can be changed from the default of `9999` with environment variable `REACT_APP_PLATFORM_PORT`.
-
-## Building a release
-
-1. Run `make generate` to create generated files
-2. Run `make ui` to compile the frontend
-3. Run `make build` to build the executable for your current platform
-
-## Cross compiling
-
-This project uses a modification of the [CI-GoReleaser](https://github.com/bep/dockerfiles/tree/master/ci-goreleaser) docker container to create an environment
-where the app can be cross-compiled.  This process is kicked off by CI via the `scripts/cross-compile.sh` script.  Run the following
-command to open a bash shell to the container to poke around:
-
-`docker run --rm --mount type=bind,source="$(pwd)",target=/stash -w /stash -i -t stashappdev/compiler:latest /bin/bash`
-
-## Profiling
-
-Stash can be profiled using the `--cpuprofile <output profile filename>` command line flag.
-
-The resulting file can then be used with pprof as follows:
-
-`go tool pprof <path to binary> <path to profile filename>`
-
-With `graphviz` installed and in the path, a call graph can be generated with:
-
-`go tool pprof -svg <path to binary> <path to profile filename> > <output svg file>`
+See [Development](docs/DEVELOPMENT.md) and [Contributing](docs/CONTRIBUTING.md) for information on working with the codebase, getting a local development setup, and contributing changes.

docker/build/x86_64/Dockerfile

@@ -1,30 +1,23 @@
-# This dockerfile must be built from the top-level stash directory
-# ie from top-level stash:
-# docker build -t stash/build -f docker/build/x86_64/Dockerfile .
+# This dockerfile should be built with `make docker-build` from the stash root.
 
 # Build Frontend
 FROM node:alpine as frontend
-RUN apk add --no-cache make git
+RUN apk add --no-cache make
 ## cache node_modules separately
 COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/
 WORKDIR /stash
 RUN yarn --cwd ui/v2.5 install --frozen-lockfile.
 COPY Makefile /stash/
-COPY ./.git /stash/.git
 COPY ./graphql /stash/graphql/
 COPY ./ui /stash/ui/
 RUN make generate-frontend
+ARG GITHASH
+ARG STASH_VERSION
 RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui
 
 # Build Backend
 FROM golang:1.17-alpine as backend
-RUN apk add --no-cache xz make alpine-sdk
-## install ffmpeg
-WORKDIR /
-RUN wget -O /ffmpeg.tar.xz https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \
-    tar xf /ffmpeg.tar.xz && \
-    rm ffmpeg.tar.xz && \
-    mv /ffmpeg*/ /ffmpeg/
+RUN apk add --no-cache make alpine-sdk
 WORKDIR /stash
 COPY ./go* ./*.go Makefile gqlgen.yml .gqlgenc.yml /stash/
 COPY ./scripts /stash/scripts/
@@ -32,12 +25,14 @@ COPY ./vendor /stash/vendor/
 COPY ./pkg /stash/pkg/
 COPY --from=frontend /stash /stash/
 RUN make generate-backend
+ARG GITHASH
+ARG STASH_VERSION
 RUN make build
 
 # Final Runnable Image
 FROM alpine:latest
-RUN apk add --no-cache ca-certificates vips-tools
-COPY --from=backend /stash/stash /ffmpeg/ffmpeg /ffmpeg/ffprobe /usr/bin/
+RUN apk add --no-cache ca-certificates vips-tools ffmpeg
+COPY --from=backend /stash/stash /usr/bin/
 ENV STASH_CONFIG_FILE=/root/.stash/config.yml
 EXPOSE 9999
 ENTRYPOINT ["stash"]

docker/build/x86_64/README.md

@@ -1,13 +1,13 @@
 # Introduction
 
-This dockerfile is used to build a stash docker container using the current source code.
+This dockerfile is used to build a stash docker container using the current source code. This is ideal for testing your current branch in docker. Note that it does not include python, so python-based scrapers will not work in this image. The production docker images distributed by the project contain python and the necessary packages.
 
 # Building the docker container
 
 From the top-level directory (should contain `main.go` file):
 
 ```
-docker build -t stash/build -f ./docker/build/x86_64/Dockerfile .
+make docker-build
 
 ```
 

docker/ci/x86_64/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=$BUILDPLATFORM ubuntu:20.04  AS prep
+FROM --platform=$BUILDPLATFORM alpine:latest AS binary
 ARG TARGETPLATFORM
 WORKDIR /
 COPY stash-*  /
@@ -8,15 +8,12 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm/v6" ];   then BIN=stash-pi; \
     elif [ "$TARGETPLATFORM" = "linux/amd64" ];  then BIN=stash-linux; \
     fi; \
     mv $BIN /stash
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-pip && pip3 install cloudscraper
-FROM ubuntu:20.04 as app
-run apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-mechanicalsoup ffmpeg libvips-tools && rm -rf /var/lib/apt/lists/*
-COPY --from=prep /stash /usr/bin/
-COPY --from=prep /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages
 
+FROM --platform=$TARGETPLATFORM alpine:latest AS app
+COPY --from=binary /stash /usr/bin/
+RUN apk add --no-cache ca-certificates python3 py3-requests py3-requests-toolbelt py3-lxml py3-pip ffmpeg vips-tools && pip install --no-cache-dir mechanicalsoup cloudscraper
+RUN ln -s /usr/bin/python3 /usr/bin/python
 ENV STASH_CONFIG_FILE=/root/.stash/config.yml
 
 EXPOSE 9999
 CMD ["stash"]
-

docker/ci/x86_64/docker_push.sh

@@ -10,5 +10,5 @@ done
 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
 
 # must build the image from dist directory
-docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push $DOCKER_TAGS -f docker/ci/x86_64/Dockerfile dist/
+docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 --push $DOCKER_TAGS -f docker/ci/x86_64/Dockerfile dist/
 

docker/compiler/Dockerfile

@@ -48,10 +48,20 @@ RUN mkdir -p /root/.ssh; \
     ssh-keyscan github.com > /root/.ssh/known_hosts;
 
 # Notes for self:
+
+# To test locally:
+# make generate
+# make ui
+# cd docker/compiler
+# make build
+# docker run -it -v /PATH_TO_STASH:/go/stash stashapp/compiler:latest /bin/bash
+# cd stash
+# make cross-compile-all
+# # binaries will show up in /dist
+
 # Windows:
 # GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++  go build -ldflags "-extldflags '-static'" -tags extended
 
-
 # Darwin
 # CC=o64-clang CXX=o64-clang++ GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build -tags extended
 # env goreleaser --config=goreleaser-extended.yml --skip-publish --skip-validate --rm-dist --release-notes=temp/0.48-relnotes-ready.md

docker/production/README.md

@@ -7,16 +7,6 @@ Only `docker` and `docker-compose` are required. For the most part your understa
 Installation instructions are available below, and if your distrobution's repository ships a current version of docker, you may use that.
 https://docs.docker.com/engine/install/
 
-### Docker
-Docker is effectively a cross-platform software package repository. It allows you to ship an entire environment in what's referred to as a container. Containers are intended to hold everything that is needed to run an application from one place to another, making it easy for everyone along the way to reproduce the environment.
-
-The StashApp docker container ships with everything you need to automatically build and run stash, including ffmpeg.
-
-### docker-compose
-Docker Compose lets you specify how and where to run your containers, and to manage their environment. The docker-compose.yml file in this folder gets you a fully working instance of StashApp exactly as you would need it to have a reasonable instance for testing / developing on. If you are deploying a live instance for production, a reverse proxy (such as NGINX or Traefik) is recommended, but not required.
-
-The latest version is always recommended.
-
 ### Get the docker-compose.yml file
 
 Now you can either navigate to the [docker-compose.yml](https://raw.githubusercontent.com/stashapp/stash/master/docker/production/docker-compose.yml) in the repository, or if you have curl, you can make your Linux console do it for you:
@@ -35,3 +25,13 @@ docker-compose up -d
 Installing StashApp this way will by default bind stash to port 9999. This is available in your web browser locally at http://localhost:9999 or on your network as http://YOUR-LOCAL-IP:9999
 
 Good luck and have fun!
+
+### Docker
+Docker is effectively a cross-platform software package repository. It allows you to ship an entire environment in what's referred to as a container. Containers are intended to hold everything that is needed to run an application from one place to another, making it easy for everyone along the way to reproduce the environment.
+
+The StashApp docker container ships with everything you need to automatically build and run stash, including ffmpeg.
+
+### docker-compose
+Docker Compose lets you specify how and where to run your containers, and to manage their environment. The docker-compose.yml file in this folder gets you a fully working instance of StashApp exactly as you would need it to have a reasonable instance for testing / developing on. If you are deploying a live instance for production, a reverse proxy (such as NGINX or Traefik) is recommended, but not required.
+
+The latest version is always recommended.

docker/production/x86_64/Dockerfile

@@ -1,27 +0,0 @@
-FROM ubuntu:20.04 as prep
-LABEL MAINTAINER="https://discord.gg/2TsNFKt"
-
-RUN apt-get update && \
-    apt-get -y install curl xz-utils && \
-    apt-get autoclean -y && \
-    rm -rf /var/lib/apt/lists/*
-WORKDIR /
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-# added " to end of stash-linux clause so that it doesn't pick up the arm builds
-RUN curl -L -o /stash $(curl -s https://api.github.com/repos/stashapp/stash/releases/latest | awk '/browser_download_url/ && /stash-linux/"' | sed -e 's/.*: "\(.*\)"/\1/') && \
-    chmod +x /stash
-    
-RUN curl --http1.1 -o /ffmpeg.tar.xz https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \
-    tar xf /ffmpeg.tar.xz && \
-    rm ffmpeg.tar.xz && \
-    mv /ffmpeg*/ /ffmpeg/
-
-FROM ubuntu:20.04 as app
-RUN apt-get update && apt-get -y install ca-certificates
-COPY --from=prep /stash /ffmpeg/ffmpeg /ffmpeg/ffprobe /usr/bin/
-
-ENV STASH_CONFIG_FILE=/root/.stash/config.yml
-
-EXPOSE 9999
-CMD ["stash"]

docs/DEVELOPMENT.md

@@ -0,0 +1,68 @@
+# Building from Source
+
+## Pre-requisites
+
+* [Go](https://golang.org/dl/)
+* [GolangCI](https://golangci-lint.run/) - A meta-linter which runs several linters in parallel
+  * To install, follow the [local installation instructions](https://golangci-lint.run/usage/install/#local-installation)
+* [Yarn](https://yarnpkg.com/en/docs/install) - Yarn package manager
+  * Run `yarn install --frozen-lockfile` in the `stash/ui/v2.5` folder (before running make generate for first time).
+
+NOTE: You may need to run the `go get` commands outside the project directory to avoid modifying the projects module file.
+
+## Environment
+
+### Windows
+
+1. Download and install [Go for Windows](https://golang.org/dl/)
+2. Download and install [MingW](https://sourceforge.net/projects/mingw-w64/)
+3. Search for "advanced system settings" and open the system properties dialog.
+    1. Click the `Environment Variables` button
+    2. Under system variables find the `Path`.  Edit and add `C:\Program Files\mingw-w64\*\mingw64\bin` (replace * with the correct path).
+
+NOTE: The `make` command in Windows will be `mingw32-make` with MingW.
+
+### macOS
+
+TODO
+
+
+## Commands
+
+* `make generate` - Generate Go and UI GraphQL files
+* `make build` - Builds the binary (make sure to build the UI as well... see below)
+* `make docker-build` - Locally builds and tags a complete 'stash/build' docker image
+* `make pre-ui` - Installs the UI dependencies. Only needs to be run once before building the UI for the first time, or if the dependencies are updated
+* `make fmt-ui` - Formats the UI source code
+* `make ui` - Builds the frontend
+* `make lint` - Run the linter on the backend
+* `make fmt` - Run `go fmt`
+* `make it` - Run the unit and integration tests
+* `make validate` - Run all of the tests and checks required to submit a PR
+* `make ui-start` - Runs the UI in development mode. Requires a running stash server to connect to. Stash port can be changed from the default of `9999` with environment variable `REACT_APP_PLATFORM_PORT`.
+
+## Building a release
+
+1. Run `make generate` to create generated files
+2. Run `make ui` to compile the frontend
+3. Run `make build` to build the executable for your current platform
+
+## Cross compiling
+
+This project uses a modification of the [CI-GoReleaser](https://github.com/bep/dockerfiles/tree/master/ci-goreleaser) docker container to create an environment
+where the app can be cross-compiled.  This process is kicked off by CI via the `scripts/cross-compile.sh` script.  Run the following
+command to open a bash shell to the container to poke around:
+
+`docker run --rm --mount type=bind,source="$(pwd)",target=/stash -w /stash -i -t stashappdev/compiler:latest /bin/bash`
+
+## Profiling
+
+Stash can be profiled using the `--cpuprofile <output profile filename>` command line flag.
+
+The resulting file can then be used with pprof as follows:
+
+`go tool pprof <path to binary> <path to profile filename>`
+
+With `graphviz` installed and in the path, a call graph can be generated with:
+
+`go tool pprof -svg <path to binary> <path to profile filename> > <output svg file>`

docs/readme_assets/docker_logo.svg

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="256px" height="185px" viewBox="0 0 256 185" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
+    <g>
+        <path d="M250.715745,70.4971666 C244.951102,66.4973277 231.740464,64.997388 221.412146,66.9973071 C220.211179,56.9977092 214.68673,48.2480609 205.078993,40.4983724 L199.554544,36.4985331 L195.711449,42.248302 C190.90758,49.7480006 188.505646,60.2475786 189.226226,70.2471769 C189.46642,73.7470364 190.667387,79.9967847 194.270289,85.496564 C190.90758,87.4964838 183.941971,89.996383 174.814621,89.996383 L1.15476998,89.996383 L0.674383104,91.9963028 C-1.00697093,101.9959 -1.00697093,133.244645 18.6888904,157.243681 C33.5808831,175.492947 55.6786788,184.742575 84.7420842,184.742575 C147.672763,184.742575 194.270289,154.493791 216.127891,99.7459909 C224.774854,99.9959813 243.269748,99.7459909 252.637292,80.996745 C252.877486,80.4967649 253.357872,79.4968046 255.039227,75.7469554 L256,73.7470364 L250.715745,70.4971666 L250.715745,70.4971666 Z M139.986573,0 L113.565295,0 L113.565295,24.9989952 L139.986573,24.9989952 L139.986573,0 L139.986573,0 Z M139.986573,29.9987943 L113.565295,29.9987943 L113.565295,54.9977896 L139.986573,54.9977896 L139.986573,29.9987943 L139.986573,29.9987943 Z M108.761427,29.9987943 L82.3401495,29.9987943 L82.3401495,54.9977896 L108.761427,54.9977896 L108.761427,29.9987943 L108.761427,29.9987943 Z M77.5362814,29.9987943 L51.1150037,29.9987943 L51.1150037,54.9977896 L77.5362814,54.9977896 L77.5362814,29.9987943 L77.5362814,29.9987943 Z M46.311135,59.9975886 L19.8898576,59.9975886 L19.8898576,84.9965839 L46.311135,84.9965839 L46.311135,59.9975886 L46.311135,59.9975886 Z M77.5362814,59.9975886 L51.1150037,59.9975886 L51.1150037,84.9965839 L77.5362814,84.9965839 L77.5362814,59.9975886 L77.5362814,59.9975886 Z M108.761427,59.9975886 L82.3401495,59.9975886 L82.3401495,84.9965839 L108.761427,84.9965839 L108.761427,59.9975886 L108.761427,59.9975886 Z M139.986573,59.9975886 L113.565295,59.9975886 L113.565295,84.9965839 L139.986573,84.9965839 L139.986573,59.9975886 L139.986573,59.9975886 Z M171.211719,59.9975886 L144.790441,59.9975886 L144.790441,84.9965839 L171.211719,84.9965839 L171.211719,59.9975886 L171.211719,59.9975886 Z" fill="#2396ED" fill-rule="nonzero"/>
+    </g>
+</svg>

docs/readme_assets/linux_logo.svg

@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg width="256px" height="295px" viewBox="0 0 256 295" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
+    <defs>
+        <filter x="-50%" y="-50%" width="200%" height="200%" filterUnits="objectBoundingBox" id="filter-1">
+            <feOffset dx="0" dy="0" in="SourceAlpha" result="shadowOffsetOuter1"></feOffset>
+            <feGaussianBlur stdDeviation="6.5" in="shadowOffsetOuter1" result="shadowBlurOuter1"></feGaussianBlur>
+        </filter>
+        <linearGradient x1="48.5477412%" y1="115.276174%" x2="51.0473804%" y2="41.3637237%" id="linearGradient-2">
+            <stop stop-color="#FFEED7" offset="0%"></stop>
+            <stop stop-color="#BDBFC2" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="54.4065463%" y1="2.40410545%" x2="46.1753957%" y2="90.5422349%" id="linearGradient-3">
+            <stop stop-color="#FFFFFF" stop-opacity="0.8" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="51.859653%" y1="88.2477484%" x2="47.9469396%" y2="9.74782136%" id="linearGradient-4">
+            <stop stop-color="#FFEED7" offset="0%"></stop>
+            <stop stop-color="#BDBFC2" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="49.9251097%" y1="85.4900173%" x2="49.9236843%" y2="13.8109272%" id="linearGradient-5">
+            <stop stop-color="#FFEED7" offset="0%"></stop>
+            <stop stop-color="#BDBFC2" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="53.9014071%" y1="3.10177585%" x2="45.9555354%" y2="93.8949571%" id="linearGradient-6">
+            <stop stop-color="#FFFFFF" stop-opacity="0.65" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="45.5928761%" y1="5.47459052%" x2="54.811359%" y2="93.5235162%" id="linearGradient-7">
+            <stop stop-color="#FFFFFF" stop-opacity="0.65" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="49.9844987%" y1="89.8452442%" x2="49.9844987%" y2="40.6316864%" id="linearGradient-8">
+            <stop stop-color="#FFEED7" offset="0%"></stop>
+            <stop stop-color="#BDBFC2" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="53.5047131%" y1="99.97524%" x2="42.7455968%" y2="23.5451715%" id="linearGradient-9">
+            <stop stop-color="#FFEED7" offset="0%"></stop>
+            <stop stop-color="#BDBFC2" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="49.8413363%" y1="13.2289558%" x2="50.2412612%" y2="94.6729694%" id="linearGradient-10">
+            <stop stop-color="#FFFFFF" stop-opacity="0.8" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="49.9272298%" y1="37.3270337%" x2="50.7270446%" y2="92.7824735%" id="linearGradient-11">
+            <stop stop-color="#FFFFFF" stop-opacity="0.65" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="49.8755597%" y1="2.29900584%" x2="49.8755597%" y2="81.203617%" id="linearGradient-12">
+            <stop stop-color="#FFFFFF" stop-opacity="0.65" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="49.8334391%" y1="2.27189065%" x2="49.8240398%" y2="71.7989617%" id="linearGradient-13">
+            <stop stop-color="#FFFFFF" stop-opacity="0.65" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="53.4670683%" y1="48.9213861%" x2="38.9488708%" y2="98.0999776%" id="linearGradient-14">
+            <stop stop-color="#FFA63F" offset="0%"></stop>
+            <stop stop-color="#FFFF00" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="52.3731508%" y1="143.008909%" x2="47.57909%" y2="-64.6215389%" id="linearGradient-15">
+            <stop stop-color="#FFEED7" offset="0%"></stop>
+            <stop stop-color="#BDBFC2" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="30.580815%" y1="34.0241079%" x2="65.8867024%" y2="89.175349%" id="linearGradient-16">
+            <stop stop-color="#FFA63F" offset="0%"></stop>
+            <stop stop-color="#FFFF00" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="59.5715091%" y1="-17.2155207%" x2="48.3608522%" y2="66.1184465%" id="linearGradient-17">
+            <stop stop-color="#FFFFFF" stop-opacity="0.65" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="47.7689553%" y1="1.56481301%" x2="51.3733028%" y2="104.312856%" id="linearGradient-18">
+            <stop stop-color="#FFFFFF" stop-opacity="0.65" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="43.5495626%" y1="4.5334861%" x2="57.1143288%" y2="92.8267174%" id="linearGradient-19">
+            <stop stop-color="#FFFFFF" stop-opacity="0.65" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="49.7328042%" y1="17.6085216%" x2="50.5582487%" y2="99.3854667%" id="linearGradient-20">
+            <stop stop-color="#FFA63F" offset="0%"></stop>
+            <stop stop-color="#FFFF00" offset="100%"></stop>
+        </linearGradient>
+        <linearGradient x1="50.1697217%" y1="2.89048531%" x2="49.6802359%" y2="94.1704279%" id="linearGradient-21">
+            <stop stop-color="#FFFFFF" stop-opacity="0.65" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </linearGradient>
+    </defs>
+    <g fill="none">
+        <g transform="translate(10.000000, 0.000000)">
+            <path d="M235.125423,249.358628 C235.125423,266.714271 182.507524,280.855905 117.584567,280.855905 C52.6616093,280.855905 0.0437105058,266.806099 0.0437105058,249.358628 L0.0437105058,249.358628 C0.0437105058,232.002986 52.6616093,217.861352 117.584567,217.861352 C182.507524,217.861352 235.033594,232.002986 235.125423,249.358628 L235.125423,249.358628 L235.125423,249.358628 Z" fill="#000" fill-opacity="0.2" filter="url(#filter-1)"></path>
+            <path d="M53.2125821,215.473804 C41.8258117,199.128278 39.6219206,145.867578 66.160442,113.084699 C79.2919595,97.3819748 82.6896249,86.4543483 83.6997416,71.6699125 C84.434372,54.8652433 71.8538272,4.81855066 119.237485,1.05357012 C167.263944,-2.80323922 164.600909,44.5804184 164.325423,69.6496791 C164.141765,90.7703016 179.844489,102.799874 190.680286,119.329056 C210.607135,149.632558 208.954216,201.791313 186.915306,230.074582 C158.999353,265.428667 135.123866,250.093259 119.237485,251.378862 C89.4849556,253.123609 88.4748389,268.918162 53.2125821,215.473804 L53.2125821,215.473804 Z" fill="#000000"></path>
+            <path d="M169.10052,122.451235 C177.365111,130.073025 198.76122,164.141508 164.876395,185.445788 C152.938652,192.88392 175.528535,221.167189 186.364333,207.484699 C205.556551,182.874582 193.343321,143.571858 181.772893,129.522053 C174.059275,119.604543 162.121532,115.747734 169.10052,122.451235 L169.10052,122.451235 Z" fill="url(#linearGradient-2)"></path>
+            <path d="M166.8048,117.859796 C180.395461,128.879251 205.097407,167.447344 169.008691,192.608434 C157.162777,200.413881 179.477174,225.115827 192.057718,212.535282 C235.676395,168.641119 190.955773,118.227111 175.528535,100.871469 C161.754216,85.719718 149.540987,104.360963 166.8048,117.859796 L166.8048,117.859796 Z" stroke="#000000" stroke-width="0.9773" fill="#000000"></path>
+            <path d="M147.245267,25.0208853 C146.786123,37.60143 132.919975,48.5290565 116.298963,49.5391732 C99.6779518,50.54929 86.638263,40.9990954 87.097407,28.4185507 L87.097407,28.4185507 C87.556551,15.8380059 101.422699,4.91037946 118.043711,3.90026272 C134.664722,2.98197479 147.704411,12.4403405 147.245267,25.0208853 L147.245267,25.0208853 L147.245267,25.0208853 Z" fill="url(#linearGradient-3)"></path>
+            <path d="M107.483399,54.9570721 C107.942543,63.1298347 104.085734,70.0169942 98.7596638,70.2924806 C93.4335938,70.567967 88.7503253,64.2317802 88.2911813,56.0590176 L88.2911813,56.0590176 C87.8320374,47.8862549 91.6888467,40.9990954 97.0149167,40.723609 C102.340987,40.4481226 107.024255,46.7843094 107.483399,54.9570721 L107.483399,54.9570721 L107.483399,54.9570721 Z" fill="url(#linearGradient-4)"></path>
+            <path d="M117.125423,55.5998736 C117.30908,65.0582394 123.461609,72.5882005 130.807913,72.4045429 C138.154216,72.2208853 143.93943,64.4154378 143.755773,54.8652433 L143.755773,54.8652433 C143.572115,45.4068775 137.419586,37.8769164 130.073282,38.060574 C122.726979,38.2442316 116.849936,46.1415079 117.125423,55.5998736 L117.125423,55.5998736 L117.125423,55.5998736 Z" fill="url(#linearGradient-5)"></path>
+            <path d="M123.186123,57.7119359 C123.094294,62.9461771 125.6655,67.1703016 129.063166,67.1703016 C132.369002,67.1703016 135.215695,62.9461771 135.307524,57.8037647 L135.307524,57.8037647 C135.399353,52.5695234 132.828146,48.3453989 129.430481,48.3453989 C126.032816,48.3453989 123.277952,52.5695234 123.186123,57.7119359 L123.186123,57.7119359 L123.186123,57.7119359 Z" fill="#000000"></path>
+            <path d="M101.973672,57.8037647 C102.432816,62.119718 100.779897,65.7928697 98.3923486,66.1601849 C96.0048,66.4356713 93.7090802,63.2216635 93.2499362,58.9057102 L93.2499362,58.9057102 C92.7907922,54.5897569 94.4437105,50.9166051 96.8312591,50.54929 C99.2188078,50.2738036 101.514528,53.4878114 101.973672,57.8037647 L101.973672,57.8037647 L101.973672,57.8037647 Z" fill="#000000"></path>
+            <path d="M124.563555,54.7734145 C124.288068,57.7119359 125.6655,60.0994845 127.593905,60.2831421 C129.52231,60.4667997 131.358886,58.1710798 131.634372,55.3243872 L131.634372,55.3243872 C131.909858,52.3858658 130.532426,49.9983172 128.604022,49.8146596 C126.675617,49.631002 124.839041,51.9267219 124.563555,54.7734145 L124.563555,54.7734145 L124.563555,54.7734145 Z" fill="url(#linearGradient-6)"></path>
+            <path d="M99.9534381,55.5080448 C100.228925,57.8955935 99.2188078,60.0076557 97.7495471,60.1913133 C96.2802864,60.3749709 94.9028545,58.538395 94.6273681,56.0590176 L94.6273681,56.0590176 C94.3518817,53.6714689 95.3619984,51.5594067 96.8312591,51.3757491 C98.3005198,51.1920915 99.6779518,53.1204962 99.9534381,55.5080448 L99.9534381,55.5080448 L99.9534381,55.5080448 Z" fill="url(#linearGradient-7)"></path>
+            <path d="M71.0273681,145.68392 C77.5472125,130.899485 91.4133603,104.911936 91.6888467,84.80143 C91.6888467,68.8232199 139.531648,64.9664106 143.388458,80.9446207 C147.245267,96.9228308 156.979119,120.798317 163.223477,132.368745 C169.467835,143.847344 187.558107,180.487033 168.274061,212.443453 C150.918419,240.726722 98.3005198,263.132948 70.2009089,208.586644 C60.6507144,189.669913 62.3954615,166.25357 71.0273681,145.68392 L71.0273681,145.68392 Z" fill="url(#linearGradient-8)"></path>
+            <path d="M65.1503253,134.664465 C59.5487689,145.224776 47.9783409,172.957072 76.2616093,188.108823 C106.65694,204.270691 106.565111,237.420885 70.0172514,221.626333 C36.5915704,207.39287 51.3760062,149.724387 60.7425432,135.950068 C66.8032436,126.308045 75.986123,114.46213 65.1503253,134.664465 L65.1503253,134.664465 Z" fill="url(#linearGradient-9)"></path>
+            <path d="M69.9254226,122.726722 C61.0180296,137.235671 39.7137494,171.395983 68.2725043,189.210769 C106.65694,212.810769 95.8211424,236.31894 60.7425432,215.106488 C11.3386521,185.537617 54.7736716,125.848901 74.5168623,103.07536 C97.1067455,77.5469553 78.8328156,107.758628 69.9254226,122.726722 L69.9254226,122.726722 Z" stroke="#000000" stroke-width="1.25" fill="#000000"></path>
+            <path d="M156.428146,151.285477 C156.428146,167.447344 140.90908,188.384309 114.27873,188.200652 C86.8219206,188.384309 75.1596638,167.447344 75.1596638,151.285477 C75.1596638,135.123609 93.341765,121.992092 115.747991,121.992092 C138.246045,122.08392 156.428146,135.123609 156.428146,151.285477 L156.428146,151.285477 Z" fill="url(#linearGradient-10)"></path>
+            <path d="M141.919197,100.504154 C141.643711,117.216994 130.716084,121.165632 116.941765,121.165632 C103.167446,121.165632 93.1581074,118.686255 91.9643331,100.504154 C91.9643331,89.1173833 103.167446,82.5057102 116.941765,82.5057102 C130.716084,82.4138814 141.919197,89.0255546 141.919197,100.504154 L141.919197,100.504154 Z" fill="url(#linearGradient-11)"></path>
+            <path d="M58.6304809,126.216216 C67.6297027,112.533726 86.638263,91.504932 62.2118039,129.154737 C42.3767844,160.19287 54.8655004,180.119718 61.293516,185.629446 C79.8429323,202.158628 79.1083019,213.269913 64.5075237,204.546177 C33.1939051,185.904932 39.7137494,154.499485 58.6304809,126.216216 L58.6304809,126.216216 Z" fill="url(#linearGradient-12)"></path>
+            <path d="M188.935539,131.817772 C181.130092,115.747734 156.336318,74.9757491 190.129314,122.359407 C220.89196,165.243453 199.312193,195.087811 195.455384,198.026333 C191.598574,200.964854 178.650714,206.933726 182.415695,196.557072 C186.272504,186.180418 205.372893,166.529056 188.935539,131.817772 L188.935539,131.817772 Z" fill="url(#linearGradient-13)"></path>
+            <path d="M51.8351502,258.541508 C31.2655004,247.613881 1.42114241,260.65357 12.2569401,231.084699 C14.4608311,224.381197 9.0429323,214.280029 12.5324265,207.760185 C16.6647222,199.77108 25.5721152,201.515827 30.8981852,196.189757 C36.1324265,190.680029 39.438263,181.129835 49.263944,182.599095 C58.9977961,184.068356 65.5176405,196.006099 72.3129712,210.698706 C77.3635549,221.167189 95.1783409,235.951625 93.9845665,247.70571 C92.5153058,265.704154 72.0374848,269.101819 51.8351502,258.541508 L51.8351502,258.541508 Z" stroke="#E68C3F" stroke-width="6.25" fill="url(#linearGradient-14)"></path>
+            <path d="M201.607913,189.11894 C198.485734,194.995983 185.446045,204.454348 176.72231,201.974971 C167.906746,199.587422 163.866279,186.180418 165.611026,175.987422 C167.263944,164.600652 176.72231,163.95785 188.660053,169.651235 C201.516084,175.987422 205.372893,181.313492 201.607913,189.11894 L201.607913,189.11894 Z" fill="url(#linearGradient-15)"></path>
+            <path d="M194.445267,253.490924 C209.505189,235.216994 243.022699,238.981975 220.432816,213.912714 C215.657718,208.494815 217.126979,196.924387 211.249936,191.965632 C204.362777,185.904932 196.740987,190.863687 189.761998,187.741508 C182.78301,184.343842 175.436707,177.823998 166.896629,182.415438 C158.356551,187.098706 157.438263,199.220107 156.611804,215.198317 C155.877174,226.676916 145.408691,245.869134 151.010247,256.429446 C159.091181,272.774971 180.119975,270.57108 194.445267,253.490924 L194.445267,253.490924 Z" stroke="#E68C3F" stroke-width="6.2507" fill="url(#linearGradient-16)"></path>
+            <path d="M187.925423,229.064465 C211.249936,194.628667 193.894294,194.904154 188.017251,192.241119 C182.140209,189.486255 175.987679,184.068356 169.10052,187.833337 C162.21336,191.690146 161.846045,201.607656 161.662388,214.647344 C161.386901,224.013881 153.581454,239.716605 158.264722,248.440341 C163.958107,258.633337 177.732426,243.848901 187.925423,229.064465 L187.925423,229.064465 Z" fill="url(#linearGradient-17)"></path>
+            <path d="M47.0600529,234.02322 C12.1651113,211.433337 28.5106366,203.719718 33.7448778,200.138395 C40.0810646,195.546955 40.1728934,186.731391 47.9783409,187.55785 C55.7837883,188.384309 60.375228,198.026333 65.6094693,209.964076 C69.4662786,218.504154 82.8732825,229.890924 81.8631658,239.716605 C80.5775626,251.287033 62.1199751,243.665243 47.0600529,234.02322 L47.0600529,234.02322 Z" fill="url(#linearGradient-18)"></path>
+            <path d="M199.587679,188.843453 C196.832816,193.618551 185.629703,201.148512 178.19157,199.128278 C170.569781,197.199874 167.080286,186.455905 168.641376,178.374971 C170.018808,169.192092 178.19157,168.732948 188.476395,173.324387 C199.404022,178.283142 202.801687,182.507267 199.587679,188.843453 L199.587679,188.843453 Z" fill="#000000"></path>
+            <path d="M192.057718,186.180418 C190.312971,189.486255 182.966668,194.720496 177.824255,193.343064 C172.681843,191.965632 170.110637,184.5275 170.937096,178.925944 C171.671726,172.589757 177.181454,172.222442 184.160442,175.344621 C191.690403,178.834115 194.077952,181.772636 192.057718,186.180418 L192.057718,186.180418 Z" fill="url(#linearGradient-19)"></path>
+            <path d="M97.1067455,66.3438425 C100.779897,62.9461771 109.68729,52.5695234 126.583788,63.4053211 C129.705967,65.4255546 132.277174,65.6092121 138.246045,68.1804184 C150.275617,73.1391732 144.582232,85.0769164 131.726201,89.1173833 C126.216473,90.8621304 121.257718,97.5656324 111.340209,96.9228308 C102.800131,96.4636868 100.59624,90.8621304 95.3619984,87.8317802 C86.0872903,82.597539 84.7098584,75.5267219 89.760442,71.7617413 C94.8110257,67.9967608 96.7394304,66.6193289 97.1067455,66.3438425 L97.1067455,66.3438425 Z" stroke="#E68C3F" stroke-width="3.75" fill="url(#linearGradient-20)"></path>
+            <path d="M138.429703,75.9858658 C133.379119,76.2613522 122.451493,87.1889787 110.972893,87.1889787 C99.4942942,87.1889787 92.6071346,76.5368386 90.8623875,76.5368386" stroke="#E68C3F" stroke-width="2.5"></path>
+            <path d="M102.800131,65.4255546 C104.636707,63.7726363 110.421921,59.2730254 118.043711,63.8644651 C119.696629,64.782753 121.349547,65.7928697 123.737096,67.1703016 C128.604022,70.0169942 126.216473,74.14929 120.33943,76.7204962 C117.676395,77.8224417 113.268613,80.2099904 109.962777,80.0263328 C106.289625,79.6590176 103.810247,77.2714689 101.422699,75.7103795 C96.9230879,72.7718581 97.1985743,70.2924806 99.3106366,68.364076 C100.871726,66.8948153 102.616473,65.5173833 102.800131,65.4255546 L102.800131,65.4255546 Z" fill="url(#linearGradient-21)"></path>
+        </g>
+    </g>
+</svg>

docs/readme_assets/mac_logo.svg

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg version="1.1" viewBox="0 0 42 42" xmlns="http://www.w3.org/2000/svg"><path d="m23.091 14.018v-0.342l-1.063 0.073c-0.301 0.019-0.527 0.083-0.679 0.191-0.152 0.109-0.228 0.26-0.228 0.453 0 0.188 0.075 0.338 0.226 0.449 0.15 0.112 0.352 0.167 0.604 0.167 0.161 0 0.312-0.025 0.451-0.074s0.261-0.118 0.363-0.206c0.102-0.087 0.182-0.191 0.239-0.312 0.058-0.121 0.087-0.254 0.087-0.399zm-2.091-13.768c-11.579 0-20.75 9.171-20.75 20.75 0 11.58 9.171 20.75 20.75 20.75s20.75-9.17 20.75-20.75c0-11.579-9.17-20.75-20.75-20.75zm4.028 12.299c0.098-0.275 0.236-0.511 0.415-0.707s0.394-0.347 0.646-0.453 0.533-0.159 0.842-0.159c0.279 0 0.531 0.042 0.755 0.125 0.225 0.083 0.417 0.195 0.578 0.336s0.289 0.305 0.383 0.493 0.15 0.387 0.169 0.596h-0.833c-0.021-0.115-0.059-0.223-0.113-0.322s-0.125-0.185-0.213-0.258c-0.089-0.073-0.193-0.13-0.312-0.171-0.12-0.042-0.254-0.062-0.405-0.062-0.177 0-0.338 0.036-0.481 0.107-0.144 0.071-0.267 0.172-0.369 0.302s-0.181 0.289-0.237 0.475c-0.057 0.187-0.085 0.394-0.085 0.622 0 0.236 0.028 0.448 0.085 0.634 0.056 0.187 0.136 0.344 0.24 0.473 0.103 0.129 0.228 0.228 0.373 0.296s0.305 0.103 0.479 0.103c0.285 0 0.517-0.067 0.697-0.201s0.296-0.33 0.35-0.588h0.834c-0.024 0.228-0.087 0.436-0.189 0.624s-0.234 0.348-0.396 0.481c-0.163 0.133-0.354 0.236-0.574 0.308s-0.462 0.109-0.725 0.109c-0.312 0-0.593-0.052-0.846-0.155-0.252-0.103-0.469-0.252-0.649-0.445s-0.319-0.428-0.417-0.705-0.147-0.588-0.147-0.935c-2e-3 -0.339 0.047-0.647 0.145-0.923zm-11.853-1.262h0.834v0.741h0.016c0.051-0.123 0.118-0.234 0.2-0.33 0.082-0.097 0.176-0.179 0.284-0.248 0.107-0.069 0.226-0.121 0.354-0.157 0.129-0.036 0.265-0.054 0.407-0.054 0.306 0 0.565 0.073 0.775 0.219 0.211 0.146 0.361 0.356 0.449 0.63h0.021c0.056-0.132 0.13-0.25 0.221-0.354s0.196-0.194 0.314-0.268 0.248-0.13 0.389-0.169 0.289-0.058 0.445-0.058c0.215 0 0.41 0.034 0.586 0.103s0.326 0.165 0.451 0.29 0.221 0.277 0.288 0.455 0.101 0.376 0.101 0.594v2.981h-0.87v-2.772c0-0.287-0.074-0.51-0.222-0.667-0.147-0.157-0.358-0.236-0.632-0.236-0.134 0-0.257 0.024-0.369 0.071-0.111 0.047-0.208 0.113-0.288 0.198-0.081 0.084-0.144 0.186-0.189 0.304-0.046 0.118-0.069 0.247-0.069 0.387v2.715h-0.858v-2.844c0-0.126-0.02-0.24-0.059-0.342s-0.094-0.189-0.167-0.262c-0.072-0.073-0.161-0.128-0.264-0.167-0.104-0.039-0.22-0.059-0.349-0.059-0.134 0-0.258 0.025-0.373 0.075-0.114 0.05-0.212 0.119-0.294 0.207-0.082 0.089-0.146 0.193-0.191 0.314-0.044 0.12-0.116 0.252-0.116 0.394v2.683h-0.825v-4.374zm1.893 20.939c-3.825 0-6.224-2.658-6.224-6.9s2.399-6.909 6.224-6.909 6.215 2.667 6.215 6.909c0 4.241-2.39 6.9-6.215 6.9zm7.082-16.575c-0.141 0.036-0.285 0.054-0.433 0.054-0.218 0-0.417-0.031-0.598-0.093-0.182-0.062-0.337-0.149-0.467-0.262s-0.232-0.249-0.304-0.409c-0.073-0.16-0.109-0.338-0.109-0.534 0-0.384 0.143-0.684 0.429-0.9s0.7-0.342 1.243-0.377l1.18-0.068v-0.338c0-0.252-0.08-0.445-0.24-0.576s-0.386-0.197-0.679-0.197c-0.118 0-0.229 0.015-0.331 0.044-0.102 0.03-0.192 0.072-0.27 0.127s-0.143 0.121-0.193 0.198c-0.051 0.076-0.086 0.162-0.105 0.256h-0.818c5e-3 -0.193 0.053-0.372 0.143-0.536s0.212-0.306 0.367-0.427 0.336-0.215 0.546-0.282 0.438-0.101 0.685-0.101c0.266 0 0.507 0.033 0.723 0.101s0.401 0.163 0.554 0.288 0.271 0.275 0.354 0.451 0.125 0.373 0.125 0.59v3.001h-0.833v-0.729h-0.021c-0.062 0.118-0.14 0.225-0.235 0.32-0.096 0.095-0.203 0.177-0.322 0.244-0.12 0.067-0.25 0.119-0.391 0.155zm5.503 16.575c-2.917 0-4.9-1.528-5.038-3.927h1.899c0.148 1.371 1.473 2.279 3.288 2.279 1.741 0 2.992-0.908 2.992-2.149 0-1.074-0.76-1.723-2.519-2.167l-1.714-0.426c-2.464-0.611-3.584-1.732-3.584-3.575 0-2.269 1.982-3.844 4.807-3.844 2.76 0 4.686 1.584 4.76 3.862h-1.88c-0.13-1.371-1.25-2.214-2.918-2.214-1.658 0-2.806 0.852-2.806 2.084 0 0.972 0.722 1.547 2.482 1.991l1.445 0.361c2.751 0.667 3.881 1.751 3.881 3.696-1e-3 2.482-1.964 4.029-5.095 4.029zm-12.585-12.106c-2.621 0-4.26 2.01-4.26 5.205 0 3.186 1.639 5.196 4.26 5.196 2.612 0 4.26-2.01 4.26-5.196 1e-3 -3.195-1.648-5.205-4.26-5.205z"/></svg>

docs/readme_assets/windows_logo.svg

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg width="256px" height="257px" viewBox="0 0 256 257" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
+    <g>
+        <path d="M0,36.3573818 L104.619084,22.1093454 L104.664817,123.02292 L0.0955693151,123.618411 L0,36.3573818 Z M104.569248,134.650129 L104.650452,235.651651 L0.0812046021,221.274919 L0.0753414539,133.972642 L104.569248,134.650129 Z M117.25153,20.2454506 L255.967753,6.21724894e-15 L255.967753,121.739477 L117.25153,122.840723 L117.25153,20.2454506 Z M256,135.599959 L255.96746,256.791232 L117.251237,237.213007 L117.056874,135.373055 L256,135.599959 Z" fill="#00ADEF"/>
+    </g>
+</svg>

go.mod

@@ -19,39 +19,42 @@ require (
 	github.com/h2non/filetype v1.0.8
 	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a
 	github.com/jmoiron/sqlx v1.3.1
-	github.com/json-iterator/go v1.1.9
+	github.com/json-iterator/go v1.1.11
 	github.com/mattn/go-sqlite3 v1.14.6
 	github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac
 	github.com/rs/cors v1.6.0
 	github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f
 	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/afero v1.2.0 // indirect
-	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.7.0
-	github.com/stretchr/testify v1.6.1
-	github.com/tidwall/gjson v1.8.1
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.9.0
+	github.com/stretchr/testify v1.7.0
+	github.com/tidwall/gjson v1.9.3
 	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/vektah/gqlparser/v2 v2.0.1
 	github.com/vektra/mockery/v2 v2.2.1
-	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
 	golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb
 	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
+	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf
 	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
-	golang.org/x/tools v0.1.0 // indirect
+	golang.org/x/text v0.3.6
+	golang.org/x/tools v0.1.5 // indirect
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 )
 
+require github.com/vektah/gqlparser/v2 v2.0.1
+
 require (
 	github.com/agnivade/levenshtein v1.1.0 // indirect
 	github.com/antchfx/xpath v1.1.6 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fsnotify/fsnotify v1.4.7 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.1.0-rc.5 // indirect
@@ -62,34 +65,33 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matryer/moq v0.0.0-20200106131100-75d0ddfc0007 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/mitchellh/mapstructure v1.4.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
-	github.com/pelletier/go-toml v1.7.0 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rs/zerolog v1.18.0 // indirect
 	github.com/russross/blackfriday/v2 v2.0.1 // indirect
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
-	github.com/spf13/cast v1.3.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/cobra v1.0.0 // indirect
-	github.com/spf13/jwalterweatherman v1.0.0 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/tidwall/match v1.0.3 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
 	github.com/urfave/cli/v2 v2.1.1 // indirect
 	github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e // indirect
-	go.uber.org/atomic v1.6.0 // indirect
-	golang.org/x/mod v0.4.1 // indirect
-	golang.org/x/text v0.3.6 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	gopkg.in/ini.v1 v1.51.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+	gopkg.in/ini.v1 v1.63.2 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
 
 replace git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999

go.sum

@@ -18,6 +18,11 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW
 cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
 cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
 cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
+cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY=
+cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
+cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
+cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
+cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -27,6 +32,7 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -50,7 +56,6 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
@@ -78,6 +83,7 @@ github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz
 github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
 github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
 github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/arrow/go/arrow v0.0.0-20200601151325-b2287a20f230/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0=
 github.com/apache/arrow/go/arrow v0.0.0-20210521153258-78c88a9f517b/go.mod h1:R4hW3Ug0s+n4CUsWHKOj00Pu01ZqU4x/hSF5kXUcXKQ=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
@@ -86,6 +92,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go-v2 v1.3.2/go.mod h1:7OaACgj2SX3XGWnrIjGlJM22h6yD6MEWKvm7levnnM8=
 github.com/aws/aws-sdk-go-v2 v1.6.0/go.mod h1:tI4KhsR5VkzlUa2DZAdwx7wCAYGwkZZ1H31PYrBFx1w=
@@ -139,6 +146,7 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM=
 github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
@@ -149,6 +157,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/corona10/goimagehash v1.0.3 h1:NZM518aKLmoNluluhfHGxT3LGOnrojrxhGn63DR/CZA=
 github.com/corona10/goimagehash v1.0.3/go.mod h1:VkvE0mLn84L4aF8vCb6mafVajEb6QYMHl2ZJLn0mOGI=
@@ -183,12 +192,15 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw=
 github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo=
 github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
@@ -240,6 +252,7 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
 github.com/gobwas/ws v1.1.0-rc.5 h1:QOAag7FoBaBYYHRqzqkhhd8fq5RTubvI4v3Ft/gDVVQ=
 github.com/gobwas/ws v1.1.0-rc.5/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
 github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -265,6 +278,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -302,14 +316,16 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github/v35 v35.2.0/go.mod h1:s0515YVTI+IMrDoy9Y4pHt9ShGpzHvHO8rZ7L7acgvs=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -321,14 +337,17 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
@@ -347,20 +366,25 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14=
 github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -373,8 +397,11 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@@ -439,12 +466,11 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
@@ -462,8 +488,10 @@ github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
@@ -478,8 +506,9 @@ github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
 github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
@@ -498,13 +527,17 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -514,8 +547,9 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo=
+github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -546,21 +580,26 @@ github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKw
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
+github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
+github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pierrec/lz4/v4 v4.1.4/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.7/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
 github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
+github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
+github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -578,6 +617,7 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq
 github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac h1:kYPjbEN6YPYWWHI6ky1J813KzIq/8+Wg4TO4xU7A/KU=
 github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -592,6 +632,7 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
+github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
@@ -613,29 +654,31 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/snowflakedb/gosnowflake v1.4.3/go.mod h1:1kyg2XEduwti88V11PKRHImhXLK5WpGiayY6lFNYb98=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.0 h1:O9FblXGxoTc51M+cqr74Bm2Tmt4PvkA5iu/j8HrkNuY=
-github.com/spf13/afero v1.2.0/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
+github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
+github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
-github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk=
+github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
@@ -646,16 +689,16 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tidwall/gjson v1.8.1 h1:8j5EE9Hrh3l9Od1OIEDAb7IpezNA20UdRngNAj5N0WU=
-github.com/tidwall/gjson v1.8.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
-github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
-github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/gjson v1.9.3 h1:hqzS9wAHMO+KVBBkLxYdkEeeFHuqr95GfClRLKlgK0E=
+github.com/tidwall/gjson v1.9.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -683,9 +726,13 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
 go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -694,15 +741,19 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -713,6 +764,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -720,8 +772,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
-golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
-golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -747,8 +799,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI=
 golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -758,8 +810,9 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -783,6 +836,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -807,6 +861,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -823,6 +879,10 @@ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -860,7 +920,10 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -869,6 +932,7 @@ golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -896,13 +960,22 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210521090106-6ca3eb03dfc2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
+golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k=
+golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -943,6 +1016,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -984,8 +1058,13 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1015,6 +1094,12 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR
 google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
 google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
 google.golang.org/api v0.45.0/go.mod h1:ISLIJCedJolbZvDfAk+Ctuq5hf+aJ33WgtUsfyFoLXA=
+google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo=
+google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4=
+google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
+google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
+google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
+google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
 google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1047,6 +1132,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
@@ -1067,6 +1153,18 @@ google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
 google.golang.org/genproto v0.0.0-20210427215850-f767ed18ee4d/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
+google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
+google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
+google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
+google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
+google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
+google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w=
+google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1082,12 +1180,19 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
+google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
+google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1100,6 +1205,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1110,8 +1216,9 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c=
+gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
 gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
@@ -1119,14 +1226,16 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg=
 gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
 gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=

graphql/documents/data/config.graphql

@@ -52,12 +52,20 @@ fragment ConfigInterfaceData on ConfigInterfaceResult {
   wallShowTitle
   wallPlayback
   maximumLoopDuration
+  noBrowser
   autostartVideo
+  autostartVideoOnPlaySelected
+  continuePlaylistDefault
   showStudioAsText
   css
   cssEnabled
   language
   slideshowDelay
+  disabledDropdownCreate {
+    performer
+    tag
+    studio
+  }
   handyKey
   funscriptOffset
 }
@@ -76,6 +84,46 @@ fragment ConfigScrapingData on ConfigScrapingResult {
   excludeTagPatterns
 }
 
+fragment IdentifyFieldOptionsData on IdentifyFieldOptions {
+  field
+  strategy
+  createMissing
+}
+
+fragment IdentifyMetadataOptionsData on IdentifyMetadataOptions {
+  fieldOptions {
+    ...IdentifyFieldOptionsData
+  }
+  setCoverImage
+  setOrganized
+  includeMalePerformers
+}
+
+fragment ScraperSourceData on ScraperSource {
+  stash_box_index
+  stash_box_endpoint
+  scraper_id
+}
+
+fragment ConfigDefaultSettingsData on ConfigDefaultSettingsResult {
+  identify {
+    sources {
+      source {
+        ...ScraperSourceData
+      }
+      options {
+        ...IdentifyMetadataOptionsData
+      }
+    }
+    options {
+      ...IdentifyMetadataOptionsData
+    }
+  }
+
+  deleteFile
+  deleteGenerated
+}
+
 fragment ConfigData on ConfigResult {
   general {
     ...ConfigGeneralData
@@ -89,4 +137,7 @@ fragment ConfigData on ConfigResult {
   scraping {
     ...ConfigScrapingData
   }
+  defaults {
+    ...ConfigDefaultSettingsData
+  }
 }

graphql/documents/mutations/config.graphql

@@ -30,6 +30,12 @@ mutation ConfigureScraping($input: ConfigScrapingInput!) {
   }
 }
 
+mutation ConfigureDefaults($input: ConfigDefaultSettingsInput!) {
+  configureDefaults(input: $input) {
+    ...ConfigDefaultSettingsData
+  }
+}
+
 mutation GenerateAPIKey($input: GenerateAPIKeyInput!) {
   generateAPIKey(input: $input)
 }

graphql/documents/mutations/metadata.graphql

@@ -26,6 +26,10 @@ mutation MetadataAutoTag($input: AutoTagMetadataInput!) {
   metadataAutoTag(input: $input)
 }
 
+mutation MetadataIdentify($input: IdentifyMetadataInput!) {
+  metadataIdentify(input: $input)
+}
+
 mutation MetadataClean($input: CleanMetadataInput!) {
   metadataClean(input: $input)
 }

graphql/documents/queries/image.graphql

@@ -1,6 +1,8 @@
 query FindImages($filter: FindFilterType, $image_filter: ImageFilterType, $image_ids: [Int!]) {
   findImages(filter: $filter, image_filter: $image_filter, image_ids: $image_ids) {
     count
+    megapixels
+    filesize
     images {
       ...SlimImageData
     }

graphql/documents/queries/scene.graphql

@@ -1,6 +1,8 @@
 query FindScenes($filter: FindFilterType, $scene_filter: SceneFilterType, $scene_ids: [Int!]) {
   findScenes(filter: $filter, scene_filter: $scene_filter, scene_ids: $scene_ids) {
     count
+    filesize
+    duration
     scenes {
       ...SlimSceneData
     }
@@ -10,6 +12,8 @@ query FindScenes($filter: FindFilterType, $scene_filter: SceneFilterType, $scene
 query FindScenesByPathRegex($filter: FindFilterType) {
   findScenesByPathRegex(filter: $filter) {
     count
+    filesize
+    duration
     scenes {
       ...SlimSceneData
     }

graphql/schema/schema.graphql

@@ -7,7 +7,7 @@ type Query {
   """Find a scene by ID or Checksum"""
   findScene(id: ID, checksum: String): Scene
   findSceneByHash(input: SceneHashInput!): Scene
-  
+
   """A function which queries Scene objects"""
   findScenes(scene_filter: SceneFilterType, scene_ids: [Int!], filter: FindFilterType): FindScenesResultType!
 
@@ -25,7 +25,7 @@ type Query {
   findSceneMarkers(scene_marker_filter: SceneMarkerFilterType filter: FindFilterType): FindSceneMarkersResultType!
 
   findImage(id: ID, checksum: String): Image
-  
+
   """A function which queries Scene objects"""
   findImages(image_filter: ImageFilterType, image_ids: [Int!], filter: FindFilterType): FindImagesResultType!
 
@@ -127,7 +127,12 @@ type Query {
   """Returns the current, complete configuration"""
   configuration: ConfigResult!
   """Returns an array of paths for the given path"""
-  directory(path: String): Directory!
+  directory(
+    "The directory path to list"
+    path: String,
+    "Desired collation locale. Determines the order of the directory result. eg. 'en-US', 'pt-BR', ..."
+    locale: String = "en"
+  ): Directory!
 
   # System status
   systemStatus: SystemStatus!
@@ -149,7 +154,7 @@ type Query {
 
   # Version
   version: Version!
-  
+
   # LatestVersion
   latestversion: ShortVersion!
 }
@@ -232,6 +237,7 @@ type Mutation {
   configureInterface(input: ConfigInterfaceInput!): ConfigInterfaceResult!
   configureDLNA(input: ConfigDLNAInput!): ConfigDLNAResult!
   configureScraping(input: ConfigScrapingInput!): ConfigScrapingResult!
+  configureDefaults(input: ConfigDefaultSettingsInput!): ConfigDefaultSettingsResult!
 
   """Generate and set (or clear) API key"""
   generateAPIKey(input: GenerateAPIKeyInput!): String!
@@ -254,6 +260,8 @@ type Mutation {
   metadataAutoTag(input: AutoTagMetadataInput!): ID!
   """Clean metadata. Returns the job ID"""
   metadataClean(input: CleanMetadataInput!): ID!
+  """Identifies scenes using scrapers. Returns the job ID"""
+  metadataIdentify(input: IdentifyMetadataInput!): ID!
   """Migrate generated files for the current hash naming"""
   migrateHashNaming: ID!
 
@@ -275,7 +283,7 @@ type Mutation {
 
   """Run batch performer tag task. Returns the job ID."""
   stashBoxBatchPerformerTag(input: StashBoxBatchPerformerTagInput!): String!
-  
+
   """Enables DLNA for an optional duration. Has no effect if DLNA is enabled by default"""
   enableDLNA(input: EnableDLNAInput!): Boolean!
   """Disables DLNA for an optional duration. Has no effect if DLNA is disabled by default"""

graphql/schema/types/config.graphql

@@ -188,56 +188,102 @@ type ConfigGeneralResult {
   stashBoxes: [StashBox!]!
 }
 
+input ConfigDisableDropdownCreateInput {
+  performer: Boolean
+  tag: Boolean
+  studio: Boolean
+}
+
 input ConfigInterfaceInput {
   """Ordered list of items that should be shown in the menu"""
   menuItems: [String!]
+
   """Enable sound on mouseover previews"""
   soundOnPreview: Boolean
+  
   """Show title and tags in wall view"""
   wallShowTitle: Boolean
   """Wall playback type"""
   wallPlayback: String
+  
   """Maximum duration (in seconds) in which a scene video will loop in the scene player"""
   maximumLoopDuration: Int
   """If true, video will autostart on load in the scene player"""
   autostartVideo: Boolean
+  """If true, video will autostart when loading from play random or play selected"""
+  autostartVideoOnPlaySelected: Boolean
+  """If true, next scene in playlist will be played at video end by default"""
+  continuePlaylistDefault: Boolean
+  
   """If true, studio overlays will be shown as text instead of logo images"""
   showStudioAsText: Boolean
+  
   """Custom CSS"""
   css: String
   cssEnabled: Boolean
+  
   """Interface language"""
   language: String
+  
   """Slideshow Delay"""
   slideshowDelay: Int
+  
+  """Set to true to disable creating new objects via the dropdown menus"""
+  disableDropdownCreate: ConfigDisableDropdownCreateInput
+  
   """Handy Connection Key"""
   handyKey: String
   """Funscript Time Offset"""
   funscriptOffset: Int
+  """True if we should not auto-open a browser window on startup"""
+  noBrowser: Boolean
+}
+
+type ConfigDisableDropdownCreate {
+  performer: Boolean!
+  tag: Boolean!
+  studio: Boolean!
 }
 
 type ConfigInterfaceResult {
   """Ordered list of items that should be shown in the menu"""
   menuItems: [String!]
+
   """Enable sound on mouseover previews"""
   soundOnPreview: Boolean
+
   """Show title and tags in wall view"""
   wallShowTitle: Boolean
   """Wall playback type"""
   wallPlayback: String
+
   """Maximum duration (in seconds) in which a scene video will loop in the scene player"""
   maximumLoopDuration: Int
+  """"True if we should not auto-open a browser window on startup"""
+  noBrowser: Boolean
   """If true, video will autostart on load in the scene player"""
   autostartVideo: Boolean
+  """If true, video will autostart when loading from play random or play selected"""
+  autostartVideoOnPlaySelected: Boolean
+  """If true, next scene in playlist will be played at video end by default"""
+  continuePlaylistDefault: Boolean
+
   """If true, studio overlays will be shown as text instead of logo images"""
   showStudioAsText: Boolean
+
   """Custom CSS"""
   css: String
   cssEnabled: Boolean
+
   """Interface language"""
   language: String
+
   """Slideshow Delay"""
   slideshowDelay: Int
+
+  """Fields are true if creating via dropdown menus are disabled"""
+  disabledDropdownCreate: ConfigDisableDropdownCreate!
+
   """Handy Connection Key"""
   handyKey: String
   """Funscript Time Offset"""
@@ -286,12 +332,31 @@ type ConfigScrapingResult {
   excludeTagPatterns: [String!]!
 }
 
+type ConfigDefaultSettingsResult {
+  identify: IdentifyMetadataTaskOptions
+  
+  """If true, delete file checkbox will be checked by default"""
+  deleteFile: Boolean
+  """If true, delete generated supporting files checkbox will be checked by default"""
+  deleteGenerated: Boolean
+}
+
+input ConfigDefaultSettingsInput {
+  identify: IdentifyMetadataInput
+
+  """If true, delete file checkbox will be checked by default"""
+  deleteFile: Boolean
+  """If true, delete generated files checkbox will be checked by default"""
+  deleteGenerated: Boolean
+}
+
 """All configuration settings"""
 type ConfigResult {
   general: ConfigGeneralResult!
   interface: ConfigInterfaceResult!
   dlna: ConfigDLNAResult!
   scraping: ConfigScrapingResult!
+  defaults: ConfigDefaultSettingsResult!
 }
 
 """Directory structure of a path"""

graphql/schema/types/gallery.graphql

@@ -74,6 +74,11 @@ input BulkGalleryUpdateInput {
 
 input GalleryDestroyInput {
   ids: [ID!]!
+  """
+  If true, then the zip file will be deleted if the gallery is zip-file-based. 
+  If gallery is folder-based, then any files not associated with other
+  galleries will be deleted, along with the folder, if it is not empty.
+  """
   delete_file: Boolean
   delete_generated: Boolean
 }

graphql/schema/types/image.graphql

@@ -70,5 +70,9 @@ input ImagesDestroyInput {
 
 type FindImagesResultType {
   count: Int!
+  """Total megapixels of the images"""
+  megapixels: Float!
+  """Total file size in bytes"""
+  filesize: Float!
   images: [Image!]!
 }

graphql/schema/types/logging.graphql

@@ -2,6 +2,7 @@
 scalar Time
 
 enum LogLevel {
+    Trace
     Debug
     Info
     Progress

graphql/schema/types/metadata.graphql

@@ -67,6 +67,88 @@ input AutoTagMetadataInput {
   tags: [String!]
 }
 
+enum IdentifyFieldStrategy {
+  """Never sets the field value"""
+  IGNORE
+  """
+  For multi-value fields, merge with existing. 
+  For single-value fields, ignore if already set
+  """
+  MERGE
+  """Always replaces the value if a value is found. 
+  For multi-value fields, any existing values are removed and replaced with the
+  scraped values.
+  """
+  OVERWRITE
+}
+
+input IdentifyFieldOptionsInput {
+  field: String!
+  strategy: IdentifyFieldStrategy!
+  """creates missing objects if needed - only applicable for performers, tags and studios"""
+  createMissing: Boolean
+}
+
+input IdentifyMetadataOptionsInput {
+  """any fields missing from here are defaulted to MERGE and createMissing false"""
+  fieldOptions: [IdentifyFieldOptionsInput!]
+  """defaults to true if not provided"""
+  setCoverImage: Boolean
+  setOrganized: Boolean
+  """defaults to true if not provided"""
+  includeMalePerformers: Boolean
+}
+
+input IdentifySourceInput {
+  source: ScraperSourceInput!
+  """Options defined for a source override the defaults"""
+  options: IdentifyMetadataOptionsInput
+}
+
+input IdentifyMetadataInput {
+  """An ordered list of sources to identify items with. Only the first source that finds a match is used."""
+  sources: [IdentifySourceInput!]!
+  """Options defined here override the configured defaults"""
+  options: IdentifyMetadataOptionsInput
+
+  """scene ids to identify"""
+  sceneIDs: [ID!]
+
+  """paths of scenes to identify - ignored if scene ids are set"""
+  paths: [String!]
+}
+
+# types for default options
+type IdentifyFieldOptions {
+  field: String!
+  strategy: IdentifyFieldStrategy!
+  """creates missing objects if needed - only applicable for performers, tags and studios"""
+  createMissing: Boolean
+}
+
+type IdentifyMetadataOptions {
+  """any fields missing from here are defaulted to MERGE and createMissing false"""
+  fieldOptions: [IdentifyFieldOptions!]
+  """defaults to true if not provided"""
+  setCoverImage: Boolean
+  setOrganized: Boolean
+  """defaults to true if not provided"""
+  includeMalePerformers: Boolean
+}
+
+type IdentifySource {
+  source: ScraperSource!
+  """Options defined for a source override the defaults"""
+  options: IdentifyMetadataOptions
+}
+
+type IdentifyMetadataTaskOptions {
+  """An ordered list of sources to identify items with. Only the first source that finds a match is used."""
+  sources: [IdentifySource!]!
+  """Options defined here override the configured defaults"""
+  options: IdentifyMetadataOptions
+}
+
 input ExportObjectTypeInput {
   ids: [String!]
   all: Boolean

graphql/schema/types/scene.graphql

@@ -120,6 +120,10 @@ input ScenesDestroyInput {
 
 type FindScenesResultType {
   count: Int!
+  """Total duration in seconds"""
+  duration: Float!
+  """Total file size in bytes"""
+  filesize: Float!
   scenes: [Scene!]!
 }
 

graphql/schema/types/scraper.graphql

@@ -31,6 +31,7 @@ type ScrapedStudio {
   stored_id: ID
   name: String!
   url: String
+  image: String
 
   remote_site_id: String
 }
@@ -95,7 +96,18 @@ input ScrapedGalleryInput {
 
 input ScraperSourceInput {
   """Index of the configured stash-box instance to use. Should be unset if scraper_id is set"""
-  stash_box_index: Int
+  stash_box_index: Int @deprecated(reason: "use stash_box_endpoint")
+  """Stash-box endpoint"""
+  stash_box_endpoint: String
+  """Scraper ID to scrape with. Should be unset if stash_box_index is set"""
+  scraper_id: ID
+}
+
+type ScraperSource {
+  """Index of the configured stash-box instance to use. Should be unset if scraper_id is set"""
+  stash_box_index: Int @deprecated(reason: "use stash_box_endpoint")
+  """Stash-box endpoint"""
+  stash_box_endpoint: String
   """Scraper ID to scrape with. Should be unset if stash_box_index is set"""
   scraper_id: ID
 }
@@ -175,10 +187,16 @@ type StashBoxFingerprint {
   duration: Int!
 }
 
+"""If neither performer_ids nor performer_names are set, tag all performers"""
 input StashBoxBatchPerformerTagInput {
+  "Stash endpoint to use for the performer tagging"
   endpoint: Int!
+  "Fields to exclude when executing the performer tagging"
   exclude_fields: [String!]
+  "Refresh performers already tagged by StashBox if true. Only tag performers with no StashBox tagging if false"
   refresh: Boolean!
+  "If set, only tag these performer ids"
   performer_ids: [ID!]
+  "If set, only tag these performer names"
   performer_names: [String!]
 }

graphql/stash-box/query.graphql

@@ -49,6 +49,7 @@ fragment PerformerFragment on Performer {
   disambiguation
   aliases
   gender
+  merged_ids
   urls {
     ...URLFragment
   }
@@ -75,11 +76,6 @@ fragment PerformerFragment on Performer {
   piercings {
     ...BodyModificationFragment
   }
-  details
-  death_date {
-    ...FuzzyDateFragment
-  }
-  weight
 }
 
 fragment PerformerAppearanceFragment on PerformerAppearance {
@@ -127,8 +123,8 @@ query FindSceneByFingerprint($fingerprint: FingerprintQueryInput!) {
   }
 }
 
-query FindScenesByFingerprints($fingerprints: [String!]!) {
-  findScenesByFingerprints(fingerprints: $fingerprints) {
+query FindScenesByFullFingerprints($fingerprints: [FingerprintQueryInput!]!) {
+  findScenesByFullFingerprints(fingerprints: $fingerprints) {
     ...SceneFragment
   }
 }
@@ -151,6 +147,12 @@ query FindPerformerByID($id: ID!) {
   }
 }
 
+query FindSceneByID($id: ID!) {
+  findScene(id: $id) {
+    ...SceneFragment
+  }
+}
+
 mutation SubmitFingerprint($input: FingerprintSubmission!) {
   submitFingerprint(input: $input)
 }

pkg/api/authentication.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"errors"
 	"net"
 	"net/http"
 	"net/url"
@@ -39,7 +40,7 @@ func authenticateHandler() func(http.Handler) http.Handler {
 
 			userID, err := manager.GetInstance().SessionStore.Authenticate(w, r)
 			if err != nil {
-				if err != session.ErrUnauthorized {
+				if errors.Is(err, session.ErrUnauthorized) {
 					w.WriteHeader(http.StatusInternalServerError)
 					_, err = w.Write([]byte(err.Error()))
 					if err != nil {
@@ -55,16 +56,18 @@ func authenticateHandler() func(http.Handler) http.Handler {
 			}
 
 			if err := session.CheckAllowPublicWithoutAuth(c, r); err != nil {
-				switch err := err.(type) {
-				case session.ExternalAccessError:
-					securityActivateTripwireAccessedFromInternetWithoutAuth(c, err, w)
+				var externalAccess session.ExternalAccessError
+				var untrustedProxy session.UntrustedProxyError
+				switch {
+				case errors.As(err, &externalAccess):
+					securityActivateTripwireAccessedFromInternetWithoutAuth(c, externalAccess, w)
 					return
-				case session.UntrustedProxyError:
-					logger.Warnf("Rejected request from untrusted proxy: %s", net.IP(err).String())
+				case errors.As(err, &untrustedProxy):
+					logger.Warnf("Rejected request from untrusted proxy: %v", net.IP(untrustedProxy))
 					w.WriteHeader(http.StatusForbidden)
 					return
 				default:
-					logger.Errorf("Error checking external access security: %s", err.Error())
+					logger.Errorf("Error checking external access security: %v", err)
 					w.WriteHeader(http.StatusInternalServerError)
 					return
 				}

pkg/api/check_version.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -15,7 +16,7 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 )
 
-//we use the github REST V3 API as no login is required
+// we use the github REST V3 API as no login is required
 const apiReleases string = "https://api.github.com/repos/stashapp/stash/releases"
 const apiTags string = "https://api.github.com/repos/stashapp/stash/tags"
 const apiAcceptHeader string = "application/vnd.github.v3+json"
@@ -107,19 +108,19 @@ type githubTagResponse struct {
 	Node_id string
 }
 
-func makeGithubRequest(url string, output interface{}) error {
+func makeGithubRequest(ctx context.Context, url string, output interface{}) error {
 	client := &http.Client{
 		Timeout: 3 * time.Second,
 	}
 
-	req, _ := http.NewRequest("GET", url, nil)
+	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 
 	req.Header.Add("Accept", apiAcceptHeader) // gh api recommendation , send header with api version
 	response, err := client.Do(req)
 
 	if err != nil {
 		//lint:ignore ST1005 Github is a proper capitalized noun
-		return fmt.Errorf("Github API request failed: %s", err)
+		return fmt.Errorf("Github API request failed: %w", err)
 	}
 
 	if response.StatusCode != http.StatusOK {
@@ -132,12 +133,12 @@ func makeGithubRequest(url string, output interface{}) error {
 	data, err := io.ReadAll(response.Body)
 	if err != nil {
 		//lint:ignore ST1005 Github is a proper capitalized noun
-		return fmt.Errorf("Github API read response failed: %s", err)
+		return fmt.Errorf("Github API read response failed: %w", err)
 	}
 
 	err = json.Unmarshal(data, output)
 	if err != nil {
-		return fmt.Errorf("unmarshalling Github API response failed: %s", err)
+		return fmt.Errorf("unmarshalling Github API response failed: %w", err)
 	}
 
 	return nil
@@ -147,7 +148,7 @@ func makeGithubRequest(url string, output interface{}) error {
 // If running a build from the "master" branch, then the latest full release
 // is used, otherwise it uses the release that is tagged with "latest_develop"
 // which is the latest pre-release build.
-func GetLatestVersion(shortHash bool) (latestVersion string, latestRelease string, err error) {
+func GetLatestVersion(ctx context.Context, shortHash bool) (latestVersion string, latestRelease string, err error) {
 
 	arch := runtime.GOARCH                                                                    // https://en.wikipedia.org/wiki/Comparison_of_ARM_cores
 	isARMv7 := cpu.ARM.HasNEON || cpu.ARM.HasVFPv3 || cpu.ARM.HasVFPv3D16 || cpu.ARM.HasVFPv4 // armv6 doesn't support any of these features
@@ -180,14 +181,14 @@ func GetLatestVersion(shortHash bool) (latestVersion string, latestRelease strin
 	}
 
 	release := githubReleasesResponse{}
-	err = makeGithubRequest(url, &release)
+	err = makeGithubRequest(ctx, url, &release)
 
 	if err != nil {
 		return "", "", err
 	}
 
 	if release.Prerelease == usePreRelease {
-		latestVersion = getReleaseHash(release, shortHash, usePreRelease)
+		latestVersion = getReleaseHash(ctx, release, shortHash, usePreRelease)
 
 		if wantedRelease != "" {
 			for _, asset := range release.Assets {
@@ -205,12 +206,12 @@ func GetLatestVersion(shortHash bool) (latestVersion string, latestRelease strin
 	return latestVersion, latestRelease, nil
 }
 
-func getReleaseHash(release githubReleasesResponse, shortHash bool, usePreRelease bool) string {
+func getReleaseHash(ctx context.Context, release githubReleasesResponse, shortHash bool, usePreRelease bool) string {
 	shaLength := len(release.Target_commitish)
 	// the /latest API call doesn't return the hash in target_commitish
 	// also add sanity check in case Target_commitish is not 40 characters
 	if !usePreRelease || shaLength != 40 {
-		return getShaFromTags(shortHash, release.Tag_name)
+		return getShaFromTags(ctx, shortHash, release.Tag_name)
 	}
 
 	if shortHash {
@@ -225,9 +226,9 @@ func getReleaseHash(release githubReleasesResponse, shortHash bool, usePreReleas
 	return release.Target_commitish
 }
 
-func printLatestVersion() {
+func printLatestVersion(ctx context.Context) {
 	_, githash, _ = GetVersion()
-	latest, _, err := GetLatestVersion(true)
+	latest, _, err := GetLatestVersion(ctx, true)
 	if err != nil {
 		logger.Errorf("Couldn't find latest version: %s", err)
 	} else {
@@ -241,13 +242,21 @@ func printLatestVersion() {
 
 // get sha from the github api tags endpoint
 // returns the sha1 hash/shorthash or "" if something's wrong
-func getShaFromTags(shortHash bool, name string) string {
+func getShaFromTags(ctx context.Context, shortHash bool, name string) string {
 	url := apiTags
 	tags := []githubTagResponse{}
-	err := makeGithubRequest(url, &tags)
+	err := makeGithubRequest(ctx, url, &tags)
 
 	if err != nil {
-		logger.Errorf("Github Tags Api %v", err)
+		// If the context is canceled, we don't want to log this as an error
+		// in the path. The function here just gives up and returns "" if
+		// something goes wrong. Hence, log the error at the info-level so
+		// it's still present, but don't treat this as an error.
+		if errors.Is(err, context.Canceled) {
+			logger.Infof("aborting sha request due to context cancellation")
+		} else {
+			logger.Errorf("Github Tags Api: %v", err)
+		}
 		return ""
 	}
 	_, gitShort, _ := GetVersion() // retrieve short hash to check actual length

pkg/api/locale.go

@@ -0,0 +1,34 @@
+package api
+
+import (
+	"golang.org/x/text/collate"
+	"golang.org/x/text/language"
+)
+
+// matcher defines a matcher for the languages we support
+var matcher = language.NewMatcher([]language.Tag{
+	language.MustParse("en-US"), // The first language is used as fallback.
+	language.MustParse("en-GB"),
+	language.MustParse("en-AU"),
+	language.MustParse("es-ES"),
+	language.MustParse("de-DE"),
+	language.MustParse("it-IT"),
+	language.MustParse("fr-FR"),
+	language.MustParse("pt-BR"),
+	language.MustParse("sv-SE"),
+	language.MustParse("zh-CN"),
+	language.MustParse("zh-TW"),
+})
+
+// newCollator parses a locale into a collator
+// Go through the available matches and return a valid match, in practice the first is a fallback
+// Optionally pass collation options through for creation.
+// If passed a nil-locale string, return nil
+func newCollator(locale *string, opts ...collate.Option) *collate.Collator {
+	if locale == nil {
+		return nil
+	}
+
+	tag, _ := language.MatchStrings(matcher, *locale)
+	return collate.New(tag, opts...)
+}

pkg/api/resolver.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"errors"
 	"sort"
 	"strconv"
 
@@ -10,6 +11,11 @@ import (
 	"github.com/stashapp/stash/pkg/plugin"
 )
 
+var (
+	ErrNotImplemented = errors.New("not implemented")
+	ErrNotSupported   = errors.New("not supported")
+)
+
 type hookExecutor interface {
 	ExecutePostHooks(ctx context.Context, id int, hookType plugin.HookTriggerEnum, input interface{}, inputFields []string)
 }
@@ -158,9 +164,9 @@ func (r *queryResolver) Version(ctx context.Context) (*models.Version, error) {
 	}, nil
 }
 
-//Gets latest version (git shorthash commit for now)
+// Latestversion returns the latest git shorthash commit.
 func (r *queryResolver) Latestversion(ctx context.Context) (*models.ShortVersion, error) {
-	ver, url, err := GetLatestVersion(true)
+	ver, url, err := GetLatestVersion(ctx, true)
 	if err == nil {
 		logger.Infof("Retrieved latest hash: %s", ver)
 	} else {

pkg/api/resolver_model_scene.go

@@ -140,7 +140,7 @@ func (r *sceneResolver) Studio(ctx context.Context, obj *models.Scene) (ret *mod
 }
 
 func (r *sceneResolver) Movies(ctx context.Context, obj *models.Scene) (ret []*models.SceneMovie, err error) {
-	if err := r.withTxn(ctx, func(repo models.Repository) error {
+	if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error {
 		qb := repo.Scene()
 		mqb := repo.Movie()
 

pkg/api/resolver_model_studio.go

@@ -39,7 +39,7 @@ func (r *studioResolver) ImagePath(ctx context.Context, obj *models.Studio) (*st
 
 	// indicate that image is missing by setting default query param to true
 	if !hasImage {
-		imagePath = imagePath + "?default=true"
+		imagePath += "?default=true"
 	}
 
 	return &imagePath, nil

pkg/api/resolver_mutation_configure.go

@@ -13,18 +13,21 @@ import (
 	"github.com/stashapp/stash/pkg/utils"
 )
 
+var ErrOverriddenConfig = errors.New("cannot set overridden value")
+
 func (r *mutationResolver) Setup(ctx context.Context, input models.SetupInput) (bool, error) {
-	err := manager.GetInstance().Setup(input)
+	err := manager.GetInstance().Setup(ctx, input)
 	return err == nil, err
 }
 
 func (r *mutationResolver) Migrate(ctx context.Context, input models.MigrateInput) (bool, error) {
-	err := manager.GetInstance().Migrate(input)
+	err := manager.GetInstance().Migrate(ctx, input)
 	return err == nil, err
 }
 
 func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.ConfigGeneralInput) (*models.ConfigGeneralResult, error) {
 	c := config.GetInstance()
+
 	existingPaths := c.GetStashPaths()
 	if len(input.Stashes) > 0 {
 		for _, s := range input.Stashes {
@@ -46,7 +49,20 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 		c.Set(config.Stash, input.Stashes)
 	}
 
-	if input.DatabasePath != nil {
+	checkConfigOverride := func(key string) error {
+		if c.HasOverride(key) {
+			return fmt.Errorf("%w: %s", ErrOverriddenConfig, key)
+		}
+
+		return nil
+	}
+
+	existingDBPath := c.GetDatabasePath()
+	if input.DatabasePath != nil && existingDBPath != *input.DatabasePath {
+		if err := checkConfigOverride(config.Database); err != nil {
+			return makeConfigGeneralResult(), err
+		}
+
 		ext := filepath.Ext(*input.DatabasePath)
 		if ext != ".db" && ext != ".sqlite" && ext != ".sqlite3" {
 			return makeConfigGeneralResult(), fmt.Errorf("invalid database path, use extension db, sqlite, or sqlite3")
@@ -54,14 +70,24 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 		c.Set(config.Database, input.DatabasePath)
 	}
 
-	if input.GeneratedPath != nil {
+	existingGeneratedPath := c.GetGeneratedPath()
+	if input.GeneratedPath != nil && existingGeneratedPath != *input.GeneratedPath {
+		if err := checkConfigOverride(config.Generated); err != nil {
+			return makeConfigGeneralResult(), err
+		}
+
 		if err := utils.EnsureDir(*input.GeneratedPath); err != nil {
 			return makeConfigGeneralResult(), err
 		}
 		c.Set(config.Generated, input.GeneratedPath)
 	}
 
-	if input.MetadataPath != nil {
+	existingMetadataPath := c.GetMetadataPath()
+	if input.MetadataPath != nil && existingMetadataPath != *input.MetadataPath {
+		if err := checkConfigOverride(config.Metadata); err != nil {
+			return makeConfigGeneralResult(), err
+		}
+
 		if *input.MetadataPath != "" {
 			if err := utils.EnsureDir(*input.MetadataPath); err != nil {
 				return makeConfigGeneralResult(), err
@@ -70,7 +96,12 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 		c.Set(config.Metadata, input.MetadataPath)
 	}
 
-	if input.CachePath != nil {
+	existingCachePath := c.GetCachePath()
+	if input.CachePath != nil && existingCachePath != *input.CachePath {
+		if err := checkConfigOverride(config.Metadata); err != nil {
+			return makeConfigGeneralResult(), err
+		}
+
 		if *input.CachePath != "" {
 			if err := utils.EnsureDir(*input.CachePath); err != nil {
 				return makeConfigGeneralResult(), err
@@ -225,17 +256,21 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 
 func (r *mutationResolver) ConfigureInterface(ctx context.Context, input models.ConfigInterfaceInput) (*models.ConfigInterfaceResult, error) {
 	c := config.GetInstance()
+
+	setBool := func(key string, v *bool) {
+		if v != nil {
+			c.Set(key, *v)
+		}
+	}
+
 	if input.MenuItems != nil {
 		c.Set(config.MenuItems, input.MenuItems)
 	}
 
-	if input.SoundOnPreview != nil {
-		c.Set(config.SoundOnPreview, *input.SoundOnPreview)
-	}
+	setBool(config.SoundOnPreview, input.SoundOnPreview)
+	setBool(config.WallShowTitle, input.WallShowTitle)
 
-	if input.WallShowTitle != nil {
-		c.Set(config.WallShowTitle, *input.WallShowTitle)
-	}
+	setBool(config.NoBrowser, input.NoBrowser)
 
 	if input.WallPlayback != nil {
 		c.Set(config.WallPlayback, *input.WallPlayback)
@@ -245,13 +280,10 @@ func (r *mutationResolver) ConfigureInterface(ctx context.Context, input models.
 		c.Set(config.MaximumLoopDuration, *input.MaximumLoopDuration)
 	}
 
-	if input.AutostartVideo != nil {
-		c.Set(config.AutostartVideo, *input.AutostartVideo)
-	}
-
-	if input.ShowStudioAsText != nil {
-		c.Set(config.ShowStudioAsText, *input.ShowStudioAsText)
-	}
+	setBool(config.AutostartVideo, input.AutostartVideo)
+	setBool(config.ShowStudioAsText, input.ShowStudioAsText)
+	setBool(config.AutostartVideoOnPlaySelected, input.AutostartVideoOnPlaySelected)
+	setBool(config.ContinuePlaylistDefault, input.ContinuePlaylistDefault)
 
 	if input.Language != nil {
 		c.Set(config.Language, *input.Language)
@@ -269,8 +301,13 @@ func (r *mutationResolver) ConfigureInterface(ctx context.Context, input models.
 
 	c.SetCSS(css)
 
-	if input.CSSEnabled != nil {
-		c.Set(config.CSSEnabled, *input.CSSEnabled)
+	setBool(config.CSSEnabled, input.CSSEnabled)
+
+	if input.DisableDropdownCreate != nil {
+		ddc := input.DisableDropdownCreate
+		setBool(config.DisableDropdownCreatePerformer, ddc.Performer)
+		setBool(config.DisableDropdownCreateStudio, ddc.Studio)
+		setBool(config.DisableDropdownCreateTag, ddc.Tag)
 	}
 
 	if input.HandyKey != nil {
@@ -350,6 +387,28 @@ func (r *mutationResolver) ConfigureScraping(ctx context.Context, input models.C
 	return makeConfigScrapingResult(), nil
 }
 
+func (r *mutationResolver) ConfigureDefaults(ctx context.Context, input models.ConfigDefaultSettingsInput) (*models.ConfigDefaultSettingsResult, error) {
+	c := config.GetInstance()
+
+	if input.Identify != nil {
+		c.Set(config.DefaultIdentifySettings, input.Identify)
+	}
+
+	if input.DeleteFile != nil {
+		c.Set(config.DeleteFileDefault, *input.DeleteFile)
+	}
+
+	if input.DeleteGenerated != nil {
+		c.Set(config.DeleteGeneratedDefault, *input.DeleteGenerated)
+	}
+
+	if err := c.Write(); err != nil {
+		return makeConfigDefaultsResult(), err
+	}
+
+	return makeConfigDefaultsResult(), nil
+}
+
 func (r *mutationResolver) GenerateAPIKey(ctx context.Context, input models.GenerateAPIKeyInput) (string, error) {
 	c := config.GetInstance()
 

pkg/api/resolver_mutation_gallery.go

@@ -441,7 +441,7 @@ func (r *mutationResolver) GalleryDestroy(ctx context.Context, input models.Gall
 						return err
 					}
 
-					if len(imgGalleries) == 0 {
+					if len(imgGalleries) == 1 {
 						if err := iqb.Destroy(img.ID); err != nil {
 							return err
 						}
@@ -465,13 +465,15 @@ func (r *mutationResolver) GalleryDestroy(ctx context.Context, input models.Gall
 	// if delete file is true, then delete the file as well
 	// if it fails, just log a message
 	if input.DeleteFile != nil && *input.DeleteFile {
-		for _, gallery := range galleries {
-			manager.DeleteGalleryFile(gallery)
-		}
-
+		// #1804 - delete the image files first, since they must be removed
+		// before deleting a folder
 		for _, img := range imgsToDelete {
 			manager.DeleteImageFile(img)
 		}
+
+		for _, gallery := range galleries {
+			manager.DeleteGalleryFile(gallery)
+		}
 	}
 
 	// if delete generated is true, then delete the generated files

pkg/api/resolver_mutation_metadata.go

@@ -90,6 +90,13 @@ func (r *mutationResolver) MetadataAutoTag(ctx context.Context, input models.Aut
 	return strconv.Itoa(jobID), nil
 }
 
+func (r *mutationResolver) MetadataIdentify(ctx context.Context, input models.IdentifyMetadataInput) (string, error) {
+	t := manager.CreateIdentifyJob(input)
+	jobID := manager.GetInstance().JobManager.Add(ctx, "Identifying...", t)
+
+	return strconv.Itoa(jobID), nil
+}
+
 func (r *mutationResolver) MetadataClean(ctx context.Context, input models.CleanMetadataInput) (string, error) {
 	jobID := manager.GetInstance().Clean(ctx, input)
 	return strconv.Itoa(jobID), nil

pkg/api/resolver_mutation_movie.go

@@ -38,7 +38,7 @@ func (r *mutationResolver) MovieCreate(ctx context.Context, input models.MovieCr
 
 	// Process the base 64 encoded image string
 	if input.FrontImage != nil {
-		frontimageData, err = utils.ProcessImageInput(*input.FrontImage)
+		frontimageData, err = utils.ProcessImageInput(ctx, *input.FrontImage)
 		if err != nil {
 			return nil, err
 		}
@@ -46,7 +46,7 @@ func (r *mutationResolver) MovieCreate(ctx context.Context, input models.MovieCr
 
 	// Process the base 64 encoded image string
 	if input.BackImage != nil {
-		backimageData, err = utils.ProcessImageInput(*input.BackImage)
+		backimageData, err = utils.ProcessImageInput(ctx, *input.BackImage)
 		if err != nil {
 			return nil, err
 		}
@@ -139,7 +139,7 @@ func (r *mutationResolver) MovieUpdate(ctx context.Context, input models.MovieUp
 	var frontimageData []byte
 	frontImageIncluded := translator.hasField("front_image")
 	if input.FrontImage != nil {
-		frontimageData, err = utils.ProcessImageInput(*input.FrontImage)
+		frontimageData, err = utils.ProcessImageInput(ctx, *input.FrontImage)
 		if err != nil {
 			return nil, err
 		}
@@ -147,7 +147,7 @@ func (r *mutationResolver) MovieUpdate(ctx context.Context, input models.MovieUp
 	backImageIncluded := translator.hasField("back_image")
 	var backimageData []byte
 	if input.BackImage != nil {
-		backimageData, err = utils.ProcessImageInput(*input.BackImage)
+		backimageData, err = utils.ProcessImageInput(ctx, *input.BackImage)
 		if err != nil {
 			return nil, err
 		}
@@ -202,7 +202,7 @@ func (r *mutationResolver) MovieUpdate(ctx context.Context, input models.MovieUp
 				// HACK - if front image is null and back image is not null, then set the front image
 				// to the default image since we can't have a null front image and a non-null back image
 				if frontimageData == nil && backimageData != nil {
-					frontimageData, _ = utils.ProcessImageInput(models.DefaultMovieImage)
+					frontimageData, _ = utils.ProcessImageInput(ctx, models.DefaultMovieImage)
 				}
 
 				if err := qb.UpdateImages(movie.ID, frontimageData, backimageData); err != nil {

pkg/api/resolver_mutation_performer.go

@@ -32,7 +32,7 @@ func (r *mutationResolver) PerformerCreate(ctx context.Context, input models.Per
 	var err error
 
 	if input.Image != nil {
-		imageData, err = utils.ProcessImageInput(*input.Image)
+		imageData, err = utils.ProcessImageInput(ctx, *input.Image)
 	}
 
 	if err != nil {
@@ -178,7 +178,7 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input models.Per
 	var err error
 	imageIncluded := translator.hasField("image")
 	if input.Image != nil {
-		imageData, err = utils.ProcessImageInput(*input.Image)
+		imageData, err = utils.ProcessImageInput(ctx, *input.Image)
 		if err != nil {
 			return nil, err
 		}

pkg/api/resolver_mutation_scene.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stashapp/stash/pkg/manager/config"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/plugin"
+	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stashapp/stash/pkg/utils"
 )
 
@@ -32,7 +33,7 @@ func (r *mutationResolver) SceneUpdate(ctx context.Context, input models.SceneUp
 
 	// Start the transaction and save the scene
 	if err := r.withTxn(ctx, func(repo models.Repository) error {
-		ret, err = r.sceneUpdate(input, translator, repo)
+		ret, err = r.sceneUpdate(ctx, input, translator, repo)
 		return err
 	}); err != nil {
 		return nil, err
@@ -52,7 +53,7 @@ func (r *mutationResolver) ScenesUpdate(ctx context.Context, input []*models.Sce
 				inputMap: inputMaps[i],
 			}
 
-			thisScene, err := r.sceneUpdate(*scene, translator, repo)
+			thisScene, err := r.sceneUpdate(ctx, *scene, translator, repo)
 			ret = append(ret, thisScene)
 
 			if err != nil {
@@ -85,7 +86,7 @@ func (r *mutationResolver) ScenesUpdate(ctx context.Context, input []*models.Sce
 	return newRet, nil
 }
 
-func (r *mutationResolver) sceneUpdate(input models.SceneUpdateInput, translator changesetTranslator, repo models.Repository) (*models.Scene, error) {
+func (r *mutationResolver) sceneUpdate(ctx context.Context, input models.SceneUpdateInput, translator changesetTranslator, repo models.Repository) (*models.Scene, error) {
 	// Populate scene from the input
 	sceneID, err := strconv.Atoi(input.ID)
 	if err != nil {
@@ -110,7 +111,7 @@ func (r *mutationResolver) sceneUpdate(input models.SceneUpdateInput, translator
 
 	if input.CoverImage != nil && *input.CoverImage != "" {
 		var err error
-		coverImageData, err = utils.ProcessImageInput(*input.CoverImage)
+		coverImageData, err = utils.ProcessImageInput(ctx, *input.CoverImage)
 		if err != nil {
 			return nil, err
 		}
@@ -119,7 +120,7 @@ func (r *mutationResolver) sceneUpdate(input models.SceneUpdateInput, translator
 	}
 
 	qb := repo.Scene()
-	scene, err := qb.Update(updatedScene)
+	s, err := qb.Update(updatedScene)
 	if err != nil {
 		return nil, err
 	}
@@ -169,13 +170,13 @@ func (r *mutationResolver) sceneUpdate(input models.SceneUpdateInput, translator
 
 	// only update the cover image if provided and everything else was successful
 	if coverImageData != nil {
-		err = manager.SetSceneScreenshot(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), coverImageData)
+		err = scene.SetScreenshot(manager.GetInstance().Paths, s.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), coverImageData)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	return scene, nil
+	return s, nil
 }
 
 func (r *mutationResolver) updateScenePerformers(qb models.SceneReaderWriter, sceneID int, performerIDs []string) error {

pkg/api/resolver_mutation_studio.go

@@ -3,10 +3,11 @@ package api
 import (
 	"context"
 	"database/sql"
-	"github.com/stashapp/stash/pkg/studio"
 	"strconv"
 	"time"
 
+	"github.com/stashapp/stash/pkg/studio"
+
 	"github.com/stashapp/stash/pkg/manager"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/plugin"
@@ -33,7 +34,7 @@ func (r *mutationResolver) StudioCreate(ctx context.Context, input models.Studio
 
 	// Process the base 64 encoded image string
 	if input.Image != nil {
-		imageData, err = utils.ProcessImageInput(*input.Image)
+		imageData, err = utils.ProcessImageInput(ctx, *input.Image)
 		if err != nil {
 			return nil, err
 		}
@@ -129,7 +130,7 @@ func (r *mutationResolver) StudioUpdate(ctx context.Context, input models.Studio
 	imageIncluded := translator.hasField("image")
 	if input.Image != nil {
 		var err error
-		imageData, err = utils.ProcessImageInput(*input.Image)
+		imageData, err = utils.ProcessImageInput(ctx, *input.Image)
 		if err != nil {
 			return nil, err
 		}

pkg/api/resolver_mutation_tag.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/plugin"
 	"github.com/stashapp/stash/pkg/tag"
@@ -36,14 +37,31 @@ func (r *mutationResolver) TagCreate(ctx context.Context, input models.TagCreate
 	var err error
 
 	if input.Image != nil {
-		imageData, err = utils.ProcessImageInput(*input.Image)
+		imageData, err = utils.ProcessImageInput(ctx, *input.Image)
 
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	// Start the transaction and save the t
+	var parentIDs []int
+	var childIDs []int
+
+	if len(input.ParentIds) > 0 {
+		parentIDs, err = utils.StringSliceToIntSlice(input.ParentIds)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(input.ChildIds) > 0 {
+		childIDs, err = utils.StringSliceToIntSlice(input.ChildIds)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Start the transaction and save the tag
 	var t *models.Tag
 	if err := r.withTxn(ctx, func(repo models.Repository) error {
 		qb := repo.Tag()
@@ -75,24 +93,22 @@ func (r *mutationResolver) TagCreate(ctx context.Context, input models.TagCreate
 			}
 		}
 
-		if input.ParentIds != nil && len(input.ParentIds) > 0 {
-			ids, err := utils.StringSliceToIntSlice(input.ParentIds)
-			if err != nil {
-				return err
-			}
-
-			if err := qb.UpdateParentTags(t.ID, ids); err != nil {
+		if len(parentIDs) > 0 {
+			if err := qb.UpdateParentTags(t.ID, parentIDs); err != nil {
 				return err
 			}
 		}
 
-		if input.ChildIds != nil && len(input.ChildIds) > 0 {
-			ids, err := utils.StringSliceToIntSlice(input.ChildIds)
-			if err != nil {
+		if len(childIDs) > 0 {
+			if err := qb.UpdateChildTags(t.ID, childIDs); err != nil {
 				return err
 			}
+		}
 
-			if err := qb.UpdateChildTags(t.ID, ids); err != nil {
+		// FIXME: This should be called before any changes are made, but
+		// requires a rewrite of ValidateHierarchy.
+		if len(parentIDs) > 0 || len(childIDs) > 0 {
+			if err := tag.ValidateHierarchy(t, parentIDs, childIDs, qb); err != nil {
 				return err
 			}
 		}
@@ -121,13 +137,30 @@ func (r *mutationResolver) TagUpdate(ctx context.Context, input models.TagUpdate
 
 	imageIncluded := translator.hasField("image")
 	if input.Image != nil {
-		imageData, err = utils.ProcessImageInput(*input.Image)
+		imageData, err = utils.ProcessImageInput(ctx, *input.Image)
 
 		if err != nil {
 			return nil, err
 		}
 	}
 
+	var parentIDs []int
+	var childIDs []int
+
+	if translator.hasField("parent_ids") {
+		parentIDs, err = utils.StringSliceToIntSlice(input.ParentIds)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if translator.hasField("child_ids") {
+		childIDs, err = utils.StringSliceToIntSlice(input.ChildIds)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Start the transaction and save the tag
 	var t *models.Tag
 	if err := r.withTxn(ctx, func(repo models.Repository) error {
@@ -183,29 +216,6 @@ func (r *mutationResolver) TagUpdate(ctx context.Context, input models.TagUpdate
 			}
 		}
 
-		var parentIDs []int
-		var childIDs []int
-
-		if translator.hasField("parent_ids") {
-			parentIDs, err = utils.StringSliceToIntSlice(input.ParentIds)
-			if err != nil {
-				return err
-			}
-		}
-
-		if translator.hasField("child_ids") {
-			childIDs, err = utils.StringSliceToIntSlice(input.ChildIds)
-			if err != nil {
-				return err
-			}
-		}
-
-		if parentIDs != nil || childIDs != nil {
-			if err := tag.EnsureUniqueHierarchy(tagID, parentIDs, childIDs, qb); err != nil {
-				return err
-			}
-		}
-
 		if parentIDs != nil {
 			if err := qb.UpdateParentTags(tagID, parentIDs); err != nil {
 				return err
@@ -218,6 +228,15 @@ func (r *mutationResolver) TagUpdate(ctx context.Context, input models.TagUpdate
 			}
 		}
 
+		// FIXME: This should be called before any changes are made, but
+		// requires a rewrite of ValidateHierarchy.
+		if parentIDs != nil || childIDs != nil {
+			if err := tag.ValidateHierarchy(t, parentIDs, childIDs, qb); err != nil {
+				logger.Errorf("Error saving tag: %s", err)
+				return err
+			}
+		}
+
 		return nil
 	}); err != nil {
 		return nil, err
@@ -317,6 +336,12 @@ func (r *mutationResolver) TagsMerge(ctx context.Context, input models.TagsMerge
 			return err
 		}
 
+		err = tag.ValidateHierarchy(t, parents, children, qb)
+		if err != nil {
+			logger.Errorf("Error merging tag: %s", err)
+			return err
+		}
+
 		return nil
 	}); err != nil {
 		return nil, err

pkg/api/resolver_query_configuration.go

@@ -6,23 +6,26 @@ import (
 	"github.com/stashapp/stash/pkg/manager/config"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/utils"
+	"golang.org/x/text/collate"
 )
 
 func (r *queryResolver) Configuration(ctx context.Context) (*models.ConfigResult, error) {
 	return makeConfigResult(), nil
 }
 
-func (r *queryResolver) Directory(ctx context.Context, path *string) (*models.Directory, error) {
+func (r *queryResolver) Directory(ctx context.Context, path, locale *string) (*models.Directory, error) {
 
 	directory := &models.Directory{}
 	var err error
 
+	col := newCollator(locale, collate.IgnoreCase, collate.Numeric)
+
 	var dirPath = ""
 	if path != nil {
 		dirPath = *path
 	}
 	currentDir := utils.GetDir(dirPath)
-	directories, err := utils.ListDir(currentDir)
+	directories, err := utils.ListDir(col, currentDir)
 	if err != nil {
 		return directory, err
 	}
@@ -40,6 +43,7 @@ func makeConfigResult() *models.ConfigResult {
 		Interface: makeConfigInterfaceResult(),
 		Dlna:      makeConfigDLNAResult(),
 		Scraping:  makeConfigScrapingResult(),
+		Defaults:  makeConfigDefaultsResult(),
 	}
 }
 
@@ -60,7 +64,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult {
 		DatabasePath:                 config.GetDatabasePath(),
 		GeneratedPath:                config.GetGeneratedPath(),
 		MetadataPath:                 config.GetMetadataPath(),
-		ConfigFilePath:               config.GetConfigFilePath(),
+		ConfigFilePath:               config.GetConfigFile(),
 		ScrapersPath:                 config.GetScrapersPath(),
 		CachePath:                    config.GetCachePath(),
 		CalculateMd5:                 config.IsCalculateMD5(),
@@ -104,8 +108,11 @@ func makeConfigInterfaceResult() *models.ConfigInterfaceResult {
 	soundOnPreview := config.GetSoundOnPreview()
 	wallShowTitle := config.GetWallShowTitle()
 	wallPlayback := config.GetWallPlayback()
+	noBrowser := config.GetNoBrowser()
 	maximumLoopDuration := config.GetMaximumLoopDuration()
 	autostartVideo := config.GetAutostartVideo()
+	autostartVideoOnPlaySelected := config.GetAutostartVideoOnPlaySelected()
+	continuePlaylistDefault := config.GetContinuePlaylistDefault()
 	showStudioAsText := config.GetShowStudioAsText()
 	css := config.GetCSS()
 	cssEnabled := config.GetCSSEnabled()
@@ -115,19 +122,23 @@ func makeConfigInterfaceResult() *models.ConfigInterfaceResult {
 	scriptOffset := config.GetFunscriptOffset()
 
 	return &models.ConfigInterfaceResult{
-		MenuItems:           menuItems,
-		SoundOnPreview:      &soundOnPreview,
-		WallShowTitle:       &wallShowTitle,
-		WallPlayback:        &wallPlayback,
-		MaximumLoopDuration: &maximumLoopDuration,
-		AutostartVideo:      &autostartVideo,
-		ShowStudioAsText:    &showStudioAsText,
-		CSS:                 &css,
-		CSSEnabled:          &cssEnabled,
-		Language:            &language,
-		SlideshowDelay:      &slideshowDelay,
-		HandyKey:            &handyKey,
-		FunscriptOffset:     &scriptOffset,
+		MenuItems:                    menuItems,
+		SoundOnPreview:               &soundOnPreview,
+		WallShowTitle:                &wallShowTitle,
+		WallPlayback:                 &wallPlayback,
+		MaximumLoopDuration:          &maximumLoopDuration,
+		NoBrowser:                    &noBrowser,
+		AutostartVideo:               &autostartVideo,
+		ShowStudioAsText:             &showStudioAsText,
+		AutostartVideoOnPlaySelected: &autostartVideoOnPlaySelected,
+		ContinuePlaylistDefault:      &continuePlaylistDefault,
+		CSS:                          &css,
+		CSSEnabled:                   &cssEnabled,
+		Language:                     &language,
+		SlideshowDelay:               &slideshowDelay,
+		DisabledDropdownCreate:       config.GetDisableDropdownCreate(),
+		HandyKey:                     &handyKey,
+		FunscriptOffset:              &scriptOffset,
 	}
 }
 
@@ -155,3 +166,15 @@ func makeConfigScrapingResult() *models.ConfigScrapingResult {
 		ExcludeTagPatterns: config.GetScraperExcludeTagPatterns(),
 	}
 }
+
+func makeConfigDefaultsResult() *models.ConfigDefaultSettingsResult {
+	config := config.GetInstance()
+	deleteFileDefault := config.GetDeleteFileDefault()
+	deleteGeneratedDefault := config.GetDeleteGeneratedDefault()
+
+	return &models.ConfigDefaultSettingsResult{
+		Identify:        config.GetDefaultIdentifySettings(),
+		DeleteFile:      &deleteFileDefault,
+		DeleteGenerated: &deleteGeneratedDefault,
+	}
+}

pkg/api/resolver_query_find_image.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"strconv"
 
+	"github.com/99designs/gqlgen/graphql"
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/utils"
 )
 
 func (r *queryResolver) FindImage(ctx context.Context, id *string, checksum *string) (*models.Image, error) {
@@ -39,14 +41,32 @@ func (r *queryResolver) FindImage(ctx context.Context, id *string, checksum *str
 func (r *queryResolver) FindImages(ctx context.Context, imageFilter *models.ImageFilterType, imageIds []int, filter *models.FindFilterType) (ret *models.FindImagesResultType, err error) {
 	if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error {
 		qb := repo.Image()
-		images, total, err := qb.Query(imageFilter, filter)
+
+		fields := graphql.CollectAllFields(ctx)
+
+		result, err := qb.Query(models.ImageQueryOptions{
+			QueryOptions: models.QueryOptions{
+				FindFilter: filter,
+				Count:      utils.StrInclude(fields, "count"),
+			},
+			ImageFilter: imageFilter,
+			Megapixels:  utils.StrInclude(fields, "megapixels"),
+			TotalSize:   utils.StrInclude(fields, "filesize"),
+		})
+		if err != nil {
+			return err
+		}
+
+		images, err := result.Resolve()
 		if err != nil {
 			return err
 		}
 
 		ret = &models.FindImagesResultType{
-			Count:  total,
-			Images: images,
+			Count:      result.Count,
+			Images:     images,
+			Megapixels: result.Megapixels,
+			Filesize:   result.TotalSize,
 		}
 
 		return nil

pkg/api/resolver_query_find_scene.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"strconv"
 
+	"github.com/99designs/gqlgen/graphql"
 	"github.com/stashapp/stash/pkg/manager"
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/utils"
 )
 
 func (r *queryResolver) FindScene(ctx context.Context, id *string, checksum *string) (*models.Scene, error) {
@@ -65,16 +67,34 @@ func (r *queryResolver) FindSceneByHash(ctx context.Context, input models.SceneH
 func (r *queryResolver) FindScenes(ctx context.Context, sceneFilter *models.SceneFilterType, sceneIDs []int, filter *models.FindFilterType) (ret *models.FindScenesResultType, err error) {
 	if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error {
 		var scenes []*models.Scene
-		var total int
 		var err error
 
+		fields := graphql.CollectAllFields(ctx)
+		result := &models.SceneQueryResult{}
+
 		if len(sceneIDs) > 0 {
 			scenes, err = repo.Scene().FindMany(sceneIDs)
 			if err == nil {
-				total = len(scenes)
+				result.Count = len(scenes)
+				for _, s := range scenes {
+					result.TotalDuration += s.Duration.Float64
+					size, _ := strconv.ParseFloat(s.Size.String, 64)
+					result.TotalSize += size
+				}
 			}
 		} else {
-			scenes, total, err = repo.Scene().Query(sceneFilter, filter)
+			result, err = repo.Scene().Query(models.SceneQueryOptions{
+				QueryOptions: models.QueryOptions{
+					FindFilter: filter,
+					Count:      utils.StrInclude(fields, "count"),
+				},
+				SceneFilter:   sceneFilter,
+				TotalDuration: utils.StrInclude(fields, "duration"),
+				TotalSize:     utils.StrInclude(fields, "filesize"),
+			})
+			if err == nil {
+				scenes, err = result.Resolve()
+			}
 		}
 
 		if err != nil {
@@ -82,8 +102,10 @@ func (r *queryResolver) FindScenes(ctx context.Context, sceneFilter *models.Scen
 		}
 
 		ret = &models.FindScenesResultType{
-			Count:  total,
-			Scenes: scenes,
+			Count:    result.Count,
+			Scenes:   scenes,
+			Duration: result.TotalDuration,
+			Filesize: result.TotalSize,
 		}
 
 		return nil
@@ -114,14 +136,31 @@ func (r *queryResolver) FindScenesByPathRegex(ctx context.Context, filter *model
 			queryFilter.Q = nil
 		}
 
-		scenes, total, err := repo.Scene().Query(sceneFilter, queryFilter)
+		fields := graphql.CollectAllFields(ctx)
+
+		result, err := repo.Scene().Query(models.SceneQueryOptions{
+			QueryOptions: models.QueryOptions{
+				FindFilter: queryFilter,
+				Count:      utils.StrInclude(fields, "count"),
+			},
+			SceneFilter:   sceneFilter,
+			TotalDuration: utils.StrInclude(fields, "duration"),
+			TotalSize:     utils.StrInclude(fields, "filesize"),
+		})
+		if err != nil {
+			return err
+		}
+
+		scenes, err := result.Resolve()
 		if err != nil {
 			return err
 		}
 
 		ret = &models.FindScenesResultType{
-			Count:  total,
-			Scenes: scenes,
+			Count:    result.Count,
+			Scenes:   scenes,
+			Duration: result.TotalDuration,
+			Filesize: result.TotalSize,
 		}
 
 		return nil

pkg/api/resolver_query_scraper.go

@@ -123,7 +123,7 @@ func (r *queryResolver) QueryStashBoxScene(ctx context.Context, input models.Sta
 	}
 
 	if input.Q != nil {
-		return client.QueryStashBoxScene(*input.Q)
+		return client.QueryStashBoxScene(ctx, *input.Q)
 	}
 
 	return nil, nil
@@ -164,18 +164,19 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.Scr
 		var singleScene *models.ScrapedScene
 		var err error
 
-		if input.SceneID != nil {
+		switch {
+		case input.SceneID != nil:
 			var sceneID int
 			sceneID, err = strconv.Atoi(*input.SceneID)
 			if err != nil {
 				return nil, err
 			}
 			singleScene, err = manager.GetInstance().ScraperCache.ScrapeScene(*source.ScraperID, sceneID)
-		} else if input.SceneInput != nil {
+		case input.SceneInput != nil:
 			singleScene, err = manager.GetInstance().ScraperCache.ScrapeSceneFragment(*source.ScraperID, *input.SceneInput)
-		} else if input.Query != nil {
+		case input.Query != nil:
 			return manager.GetInstance().ScraperCache.ScrapeSceneQuery(*source.ScraperID, *input.Query)
-		} else {
+		default:
 			err = errors.New("scene_id, scene_input or query must be set")
 		}
 
@@ -197,7 +198,7 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.Scr
 		if input.SceneID != nil {
 			return client.FindStashBoxScenesByFingerprintsFlat([]string{*input.SceneID})
 		} else if input.Query != nil {
-			return client.QueryStashBoxScene(*input.Query)
+			return client.QueryStashBoxScene(ctx, *input.Query)
 		}
 
 		return nil, errors.New("scene_id or query must be set")
@@ -208,7 +209,7 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.Scr
 
 func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeMultiScenesInput) ([][]*models.ScrapedScene, error) {
 	if source.ScraperID != nil {
-		return nil, errors.New("not implemented")
+		return nil, ErrNotImplemented
 	} else if source.StashBoxIndex != nil {
 		client, err := r.getStashBoxClient(*source.StashBoxIndex)
 		if err != nil {
@@ -240,7 +241,7 @@ func (r *queryResolver) ScrapeSinglePerformer(ctx context.Context, source models
 			return manager.GetInstance().ScraperCache.ScrapePerformerList(*source.ScraperID, *input.Query)
 		}
 
-		return nil, errors.New("not implemented")
+		return nil, ErrNotImplemented
 	} else if source.StashBoxIndex != nil {
 		client, err := r.getStashBoxClient(*source.StashBoxIndex)
 		if err != nil {
@@ -248,12 +249,13 @@ func (r *queryResolver) ScrapeSinglePerformer(ctx context.Context, source models
 		}
 
 		var ret []*models.StashBoxPerformerQueryResult
-		if input.PerformerID != nil {
+		switch {
+		case input.PerformerID != nil:
 			ret, err = client.FindStashBoxPerformersByNames([]string{*input.PerformerID})
-		} else if input.Query != nil {
+		case input.Query != nil:
 			ret, err = client.QueryStashBoxPerformer(*input.Query)
-		} else {
-			return nil, errors.New("not implemented")
+		default:
+			return nil, ErrNotImplemented
 		}
 
 		if err != nil {
@@ -272,7 +274,7 @@ func (r *queryResolver) ScrapeSinglePerformer(ctx context.Context, source models
 
 func (r *queryResolver) ScrapeMultiPerformers(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeMultiPerformersInput) ([][]*models.ScrapedPerformer, error) {
 	if source.ScraperID != nil {
-		return nil, errors.New("not implemented")
+		return nil, ErrNotImplemented
 	} else if source.StashBoxIndex != nil {
 		client, err := r.getStashBoxClient(*source.StashBoxIndex)
 		if err != nil {
@@ -290,17 +292,18 @@ func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source models.S
 		var singleGallery *models.ScrapedGallery
 		var err error
 
-		if input.GalleryID != nil {
+		switch {
+		case input.GalleryID != nil:
 			var galleryID int
 			galleryID, err = strconv.Atoi(*input.GalleryID)
 			if err != nil {
 				return nil, err
 			}
 			singleGallery, err = manager.GetInstance().ScraperCache.ScrapeGallery(*source.ScraperID, galleryID)
-		} else if input.GalleryInput != nil {
+		case input.GalleryInput != nil:
 			singleGallery, err = manager.GetInstance().ScraperCache.ScrapeGalleryFragment(*source.ScraperID, *input.GalleryInput)
-		} else {
-			return nil, errors.New("not implemented")
+		default:
+			return nil, ErrNotImplemented
 		}
 
 		if err != nil {
@@ -313,12 +316,12 @@ func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source models.S
 
 		return nil, nil
 	} else if source.StashBoxIndex != nil {
-		return nil, errors.New("not supported")
+		return nil, ErrNotSupported
 	}
 
 	return nil, errors.New("scraper_id must be set")
 }
 
 func (r *queryResolver) ScrapeSingleMovie(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSingleMovieInput) ([]*models.ScrapedMovie, error) {
-	return nil, errors.New("not supported")
+	return nil, ErrNotSupported
 }

pkg/api/resolver_subscription_logging.go

@@ -8,20 +8,22 @@ import (
 )
 
 func getLogLevel(logType string) models.LogLevel {
-	if logType == "progress" {
+	switch logType {
+	case "progress":
 		return models.LogLevelProgress
-	} else if logType == "debug" {
+	case "trace":
+		return models.LogLevelTrace
+	case "debug":
 		return models.LogLevelDebug
-	} else if logType == "info" {
+	case "info":
 		return models.LogLevelInfo
-	} else if logType == "warn" {
+	case "warn":
 		return models.LogLevelWarning
-	} else if logType == "error" {
+	case "error":
 		return models.LogLevelError
+	default:
+		return models.LogLevelDebug
 	}
-
-	// default to debug
-	return models.LogLevelDebug
 }
 
 func logEntriesFromLogItems(logItems []logger.LogItem) []*models.LogEntry {

pkg/api/routes_image.go

@@ -43,7 +43,7 @@ func (rs imageRoutes) Thumbnail(w http.ResponseWriter, r *http.Request) {
 	if exists {
 		http.ServeFile(w, r, filepath)
 	} else {
-		encoder := image.NewThumbnailEncoder(manager.GetInstance().FFMPEGPath)
+		encoder := image.NewThumbnailEncoder(manager.GetInstance().FFMPEG)
 		data, err := encoder.GetThumbnail(img, models.DefaultGthumbWidth)
 		if err != nil {
 			logger.Errorf("error generating thumbnail for image: %s", err.Error())

pkg/api/routes_scene.go

@@ -57,7 +57,8 @@ func getSceneFileContainer(scene *models.Scene) ffmpeg.Container {
 		container = ffmpeg.Container(scene.Format.String)
 	} else { // container isn't in the DB
 		// shouldn't happen, fallback to ffprobe
-		tmpVideoFile, err := ffmpeg.NewVideoFile(manager.GetInstance().FFProbePath, scene.Path, false)
+		ffprobe := manager.GetInstance().FFProbe
+		tmpVideoFile, err := ffprobe.NewVideoFile(scene.Path, false)
 		if err != nil {
 			logger.Errorf("[transcode] error reading video file: %v", err)
 			return ffmpeg.Container("")
@@ -105,7 +106,8 @@ func (rs sceneRoutes) StreamMp4(w http.ResponseWriter, r *http.Request) {
 func (rs sceneRoutes) StreamHLS(w http.ResponseWriter, r *http.Request) {
 	scene := r.Context().Value(sceneKey).(*models.Scene)
 
-	videoFile, err := ffmpeg.NewVideoFile(manager.GetInstance().FFProbePath, scene.Path, false)
+	ffprobe := manager.GetInstance().FFProbe
+	videoFile, err := ffprobe.NewVideoFile(scene.Path, false)
 	if err != nil {
 		logger.Errorf("[stream] error reading video file: %v", err)
 		return
@@ -142,8 +144,8 @@ func (rs sceneRoutes) streamTranscode(w http.ResponseWriter, r *http.Request, vi
 	scene := r.Context().Value(sceneKey).(*models.Scene)
 
 	// needs to be transcoded
-
-	videoFile, err := ffmpeg.NewVideoFile(manager.GetInstance().FFProbePath, scene.Path, false)
+	ffprobe := manager.GetInstance().FFProbe
+	videoFile, err := ffprobe.NewVideoFile(scene.Path, false)
 	if err != nil {
 		logger.Errorf("[stream] error reading video file: %v", err)
 		return
@@ -171,7 +173,7 @@ func (rs sceneRoutes) streamTranscode(w http.ResponseWriter, r *http.Request, vi
 		options.MaxTranscodeSize = models.StreamingResolutionEnum(requestedSize)
 	}
 
-	encoder := ffmpeg.NewEncoder(manager.GetInstance().FFMPEGPath)
+	encoder := manager.GetInstance().FFMPEG
 	stream, err = encoder.GetTranscodeStream(options)
 
 	if err != nil {

pkg/api/routes_studio.go

@@ -2,8 +2,10 @@ package api
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"strconv"
+	"syscall"
 
 	"github.com/go-chi/chi"
 	"github.com/stashapp/stash/pkg/logger"
@@ -47,7 +49,12 @@ func (rs studioRoutes) Image(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := utils.ServeImage(image, w, r); err != nil {
-		logger.Warnf("error serving studio image: %v", err)
+		// Broken pipe errors are common when serving images and the remote
+		// connection closes the connection. Filter them out of the error
+		// messages, as they are benign.
+		if !errors.Is(err, syscall.EPIPE) {
+			logger.Warnf("cannot serve studio image: %v", err)
+		}
 	}
 }
 

pkg/api/server.go

@@ -23,6 +23,7 @@ import (
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"
 	"github.com/gorilla/websocket"
+	"github.com/pkg/browser"
 	"github.com/rs/cors"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/manager"
@@ -34,6 +35,7 @@ import (
 var version string
 var buildstamp string
 var githash string
+var officialBuild string
 
 func Start(uiBox embed.FS, loginUIBox embed.FS) {
 	initialiseImages()
@@ -229,7 +231,7 @@ func Start(uiBox embed.FS, loginUIBox embed.FS) {
 	tlsConfig, err := makeTLSConfig(c)
 	if err != nil {
 		// assume we don't want to start with a broken TLS configuration
-		panic(fmt.Errorf("error loading TLS config: %s", err.Error()))
+		panic(fmt.Errorf("error loading TLS config: %v", err))
 	}
 
 	server := &http.Server{
@@ -240,14 +242,28 @@ func Start(uiBox embed.FS, loginUIBox embed.FS) {
 
 	go func() {
 		printVersion()
-		printLatestVersion()
+		printLatestVersion(context.TODO())
 		logger.Infof("stash is listening on " + address)
+		if tlsConfig != nil {
+			displayAddress = "https://" + displayAddress + "/"
+		} else {
+			displayAddress = "http://" + displayAddress + "/"
+		}
+
+		// This can be done before actually starting the server, as modern browsers will
+		// automatically reload the page if a local port is closed at page load and then opened.
+		if !c.GetNoBrowser() && manager.GetInstance().IsDesktop() {
+			err = browser.OpenURL(displayAddress)
+			if err != nil {
+				logger.Error("Could not open browser: " + err.Error())
+			}
+		}
 
 		if tlsConfig != nil {
-			logger.Infof("stash is running at https://" + displayAddress + "/")
+			logger.Infof("stash is running at " + displayAddress)
 			logger.Error(server.ListenAndServeTLS("", ""))
 		} else {
-			logger.Infof("stash is running at http://" + displayAddress + "/")
+			logger.Infof("stash is running at " + displayAddress)
 			logger.Error(server.ListenAndServe())
 		}
 	}()
@@ -255,12 +271,21 @@ func Start(uiBox embed.FS, loginUIBox embed.FS) {
 
 func printVersion() {
 	versionString := githash
+	if IsOfficialBuild() {
+		versionString += " - Official Build"
+	} else {
+		versionString += " - Unofficial Build"
+	}
 	if version != "" {
 		versionString = version + " (" + versionString + ")"
 	}
 	fmt.Printf("stash version: %s - %s\n", versionString, buildstamp)
 }
 
+func IsOfficialBuild() bool {
+	return officialBuild == "true"
+}
+
 func GetVersion() (string, string, string) {
 	return version, githash, buildstamp
 }
@@ -296,7 +321,7 @@ func makeTLSConfig(c *config.Instance) (*tls.Config, error) {
 	certs := make([]tls.Certificate, 1)
 	certs[0], err = tls.X509KeyPair(cert, key)
 	if err != nil {
-		return nil, fmt.Errorf("error parsing key pair: %s", err.Error())
+		return nil, fmt.Errorf("error parsing key pair: %v", err)
 	}
 	tlsConfig := &tls.Config{
 		Certificates: certs,
@@ -327,7 +352,7 @@ func BaseURLMiddleware(next http.Handler) http.Handler {
 
 		port := ""
 		forwardedPort := r.Header.Get("X-Forwarded-Port")
-		if forwardedPort != "" && forwardedPort != "80" && forwardedPort != "8080" {
+		if forwardedPort != "" && forwardedPort != "80" && forwardedPort != "8080" && forwardedPort != "443" && !strings.Contains(r.Host, ":") {
 			port = ":" + forwardedPort
 		}
 

pkg/api/session.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"embed"
+	"errors"
 	"fmt"
 	"html/template"
 	"net/http"
@@ -60,7 +61,7 @@ func handleLogin(loginUIBox embed.FS) http.HandlerFunc {
 		}
 
 		err := manager.GetInstance().SessionStore.Login(w, r)
-		if err == session.ErrInvalidCredentials {
+		if errors.Is(err, session.ErrInvalidCredentials) {
 			// redirect back to the login page with an error
 			redirectToLogin(loginUIBox, w, url, "Username or password is invalid")
 			return

pkg/autotag/gallery.go

@@ -1,78 +1,10 @@
 package autotag
 
 import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
 	"github.com/stashapp/stash/pkg/gallery"
 	"github.com/stashapp/stash/pkg/models"
 )
 
-func galleryPathsFilter(paths []string) *models.GalleryFilterType {
-	if paths == nil {
-		return nil
-	}
-
-	sep := string(filepath.Separator)
-
-	var ret *models.GalleryFilterType
-	var or *models.GalleryFilterType
-	for _, p := range paths {
-		newOr := &models.GalleryFilterType{}
-		if or != nil {
-			or.Or = newOr
-		} else {
-			ret = newOr
-		}
-
-		or = newOr
-
-		if !strings.HasSuffix(p, sep) {
-			p = p + sep
-		}
-
-		or.Path = &models.StringCriterionInput{
-			Modifier: models.CriterionModifierEquals,
-			Value:    p + "%",
-		}
-	}
-
-	return ret
-}
-
-func getMatchingGalleries(name string, paths []string, galleryReader models.GalleryReader) ([]*models.Gallery, error) {
-	regex := getPathQueryRegex(name)
-	organized := false
-	filter := models.GalleryFilterType{
-		Path: &models.StringCriterionInput{
-			Value:    "(?i)" + regex,
-			Modifier: models.CriterionModifierMatchesRegex,
-		},
-		Organized: &organized,
-	}
-
-	filter.And = galleryPathsFilter(paths)
-
-	pp := models.PerPageAll
-	gallerys, _, err := galleryReader.Query(&filter, &models.FindFilterType{
-		PerPage: &pp,
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("error querying gallerys with regex '%s': %s", regex, err.Error())
-	}
-
-	var ret []*models.Gallery
-	for _, p := range gallerys {
-		if nameMatchesPath(name, p.Path.String) {
-			ret = append(ret, p)
-		}
-	}
-
-	return ret, nil
-}
-
 func getGalleryFileTagger(s *models.Gallery) tagger {
 	return tagger{
 		ID:   s.ID,

pkg/autotag/image.go

@@ -1,78 +1,10 @@
 package autotag
 
 import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
 	"github.com/stashapp/stash/pkg/image"
 	"github.com/stashapp/stash/pkg/models"
 )
 
-func imagePathsFilter(paths []string) *models.ImageFilterType {
-	if paths == nil {
-		return nil
-	}
-
-	sep := string(filepath.Separator)
-
-	var ret *models.ImageFilterType
-	var or *models.ImageFilterType
-	for _, p := range paths {
-		newOr := &models.ImageFilterType{}
-		if or != nil {
-			or.Or = newOr
-		} else {
-			ret = newOr
-		}
-
-		or = newOr
-
-		if !strings.HasSuffix(p, sep) {
-			p = p + sep
-		}
-
-		or.Path = &models.StringCriterionInput{
-			Modifier: models.CriterionModifierEquals,
-			Value:    p + "%",
-		}
-	}
-
-	return ret
-}
-
-func getMatchingImages(name string, paths []string, imageReader models.ImageReader) ([]*models.Image, error) {
-	regex := getPathQueryRegex(name)
-	organized := false
-	filter := models.ImageFilterType{
-		Path: &models.StringCriterionInput{
-			Value:    "(?i)" + regex,
-			Modifier: models.CriterionModifierMatchesRegex,
-		},
-		Organized: &organized,
-	}
-
-	filter.And = imagePathsFilter(paths)
-
-	pp := models.PerPageAll
-	images, _, err := imageReader.Query(&filter, &models.FindFilterType{
-		PerPage: &pp,
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("error querying images with regex '%s': %s", regex, err.Error())
-	}
-
-	var ret []*models.Image
-	for _, p := range images {
-		if nameMatchesPath(name, p.Path) {
-			ret = append(ret, p)
-		}
-	}
-
-	return ret, nil
-}
-
 func getImageFileTagger(s *models.Image) tagger {
 	return tagger{
 		ID:   s.ID,

pkg/autotag/performer.go

@@ -7,25 +7,6 @@ import (
 	"github.com/stashapp/stash/pkg/scene"
 )
 
-func getMatchingPerformers(path string, performerReader models.PerformerReader) ([]*models.Performer, error) {
-	words := getPathWords(path)
-	performers, err := performerReader.QueryForAutoTag(words)
-
-	if err != nil {
-		return nil, err
-	}
-
-	var ret []*models.Performer
-	for _, p := range performers {
-		// TODO - commenting out alias handling until both sides work correctly
-		if nameMatchesPath(p.Name.String, path) { // || nameMatchesPath(p.Aliases.String, path) {
-			ret = append(ret, p)
-		}
-	}
-
-	return ret, nil
-}
-
 func getPerformerTagger(p *models.Performer) tagger {
 	return tagger{
 		ID:   p.ID,

pkg/autotag/performer_test.go

@@ -3,8 +3,10 @@ package autotag
 import (
 	"testing"
 
+	"github.com/stashapp/stash/pkg/image"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/mocks"
+	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -70,7 +72,8 @@ func testPerformerScenes(t *testing.T, performerName, expectedRegex string) {
 		PerPage: &perPage,
 	}
 
-	mockSceneReader.On("Query", expectedSceneFilter, expectedFindFilter).Return(scenes, len(scenes), nil).Once()
+	mockSceneReader.On("Query", scene.QueryOptions(expectedSceneFilter, expectedFindFilter, false)).
+		Return(mocks.SceneQueryResult(scenes, len(scenes)), nil).Once()
 
 	for i := range matchingPaths {
 		sceneID := i + 1
@@ -144,7 +147,8 @@ func testPerformerImages(t *testing.T, performerName, expectedRegex string) {
 		PerPage: &perPage,
 	}
 
-	mockImageReader.On("Query", expectedImageFilter, expectedFindFilter).Return(images, len(images), nil).Once()
+	mockImageReader.On("Query", image.QueryOptions(expectedImageFilter, expectedFindFilter, false)).
+		Return(mocks.ImageQueryResult(images, len(images)), nil).Once()
 
 	for i := range matchingPaths {
 		imageID := i + 1

pkg/autotag/scene.go

@@ -1,78 +1,10 @@
 package autotag
 
 import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
 )
 
-func scenePathsFilter(paths []string) *models.SceneFilterType {
-	if paths == nil {
-		return nil
-	}
-
-	sep := string(filepath.Separator)
-
-	var ret *models.SceneFilterType
-	var or *models.SceneFilterType
-	for _, p := range paths {
-		newOr := &models.SceneFilterType{}
-		if or != nil {
-			or.Or = newOr
-		} else {
-			ret = newOr
-		}
-
-		or = newOr
-
-		if !strings.HasSuffix(p, sep) {
-			p = p + sep
-		}
-
-		or.Path = &models.StringCriterionInput{
-			Modifier: models.CriterionModifierEquals,
-			Value:    p + "%",
-		}
-	}
-
-	return ret
-}
-
-func getMatchingScenes(name string, paths []string, sceneReader models.SceneReader) ([]*models.Scene, error) {
-	regex := getPathQueryRegex(name)
-	organized := false
-	filter := models.SceneFilterType{
-		Path: &models.StringCriterionInput{
-			Value:    "(?i)" + regex,
-			Modifier: models.CriterionModifierMatchesRegex,
-		},
-		Organized: &organized,
-	}
-
-	filter.And = scenePathsFilter(paths)
-
-	pp := models.PerPageAll
-	scenes, _, err := sceneReader.Query(&filter, &models.FindFilterType{
-		PerPage: &pp,
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("error querying scenes with regex '%s': %s", regex, err.Error())
-	}
-
-	var ret []*models.Scene
-	for _, p := range scenes {
-		if nameMatchesPath(name, p.Path) {
-			ret = append(ret, p)
-		}
-	}
-
-	return ret, nil
-}
-
 func getSceneFileTagger(s *models.Scene) tagger {
 	return tagger{
 		ID:   s.ID,

pkg/autotag/scene_test.go

@@ -67,7 +67,8 @@ func generateFalseNamePatterns(name string, separator, ext string) []string {
 }
 
 func generateTestPaths(testName, ext string) (scenePatterns []string, falseScenePatterns []string) {
-	separators := append(testSeparators, testEndSeparators...)
+	separators := testSeparators
+	separators = append(separators, testEndSeparators...)
 
 	for _, separator := range separators {
 		scenePatterns = append(scenePatterns, generateNamePatterns(testName, separator, ext)...)
@@ -79,7 +80,7 @@ func generateTestPaths(testName, ext string) (scenePatterns []string, falseScene
 	// add test cases for intra-name separators
 	for _, separator := range testSeparators {
 		if separator != " " {
-			scenePatterns = append(scenePatterns, generateNamePatterns(strings.Replace(testName, " ", separator, -1), separator, ext)...)
+			scenePatterns = append(scenePatterns, generateNamePatterns(strings.ReplaceAll(testName, " ", separator), separator, ext)...)
 		}
 	}
 
@@ -115,7 +116,8 @@ func generateTestTable(testName, ext string) []pathTestTable {
 	var scenePatterns []string
 	var falseScenePatterns []string
 
-	separators := append(testSeparators, testEndSeparators...)
+	separators := testSeparators
+	separators = append(separators, testEndSeparators...)
 
 	for _, separator := range separators {
 		scenePatterns = append(scenePatterns, generateNamePatterns(testName, separator, ext)...)

pkg/autotag/studio.go

@@ -2,46 +2,10 @@ package autotag
 
 import (
 	"database/sql"
+
 	"github.com/stashapp/stash/pkg/models"
 )
 
-func getMatchingStudios(path string, reader models.StudioReader) ([]*models.Studio, error) {
-	words := getPathWords(path)
-	candidates, err := reader.QueryForAutoTag(words)
-
-	if err != nil {
-		return nil, err
-	}
-
-	var ret []*models.Studio
-	for _, c := range candidates {
-		matches := false
-		if nameMatchesPath(c.Name.String, path) {
-			matches = true
-		}
-
-		if !matches {
-			aliases, err := reader.GetAliases(c.ID)
-			if err != nil {
-				return nil, err
-			}
-
-			for _, alias := range aliases {
-				if nameMatchesPath(alias, path) {
-					matches = true
-					break
-				}
-			}
-		}
-
-		if matches {
-			ret = append(ret, c)
-		}
-	}
-
-	return ret, nil
-}
-
 func addSceneStudio(sceneWriter models.SceneReaderWriter, sceneID, studioID int) (bool, error) {
 	// don't set if already set
 	scene, err := sceneWriter.Find(sceneID)

pkg/autotag/studio_test.go

@@ -3,8 +3,10 @@ package autotag
 import (
 	"testing"
 
+	"github.com/stashapp/stash/pkg/image"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/mocks"
+	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -111,11 +113,12 @@ func testStudioScenes(t *testing.T, tc testStudioCase) {
 	}
 
 	// if alias provided, then don't find by name
-	onNameQuery := mockSceneReader.On("Query", expectedSceneFilter, expectedFindFilter)
+	onNameQuery := mockSceneReader.On("Query", scene.QueryOptions(expectedSceneFilter, expectedFindFilter, false))
+
 	if aliasName == "" {
-		onNameQuery.Return(scenes, len(scenes), nil).Once()
+		onNameQuery.Return(mocks.SceneQueryResult(scenes, len(scenes)), nil).Once()
 	} else {
-		onNameQuery.Return(nil, 0, nil).Once()
+		onNameQuery.Return(mocks.SceneQueryResult(nil, 0), nil).Once()
 
 		expectedAliasFilter := &models.SceneFilterType{
 			Organized: &organized,
@@ -125,7 +128,8 @@ func testStudioScenes(t *testing.T, tc testStudioCase) {
 			},
 		}
 
-		mockSceneReader.On("Query", expectedAliasFilter, expectedFindFilter).Return(scenes, len(scenes), nil).Once()
+		mockSceneReader.On("Query", scene.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
+			Return(mocks.SceneQueryResult(scenes, len(scenes)), nil).Once()
 	}
 
 	for i := range matchingPaths {
@@ -202,11 +206,11 @@ func testStudioImages(t *testing.T, tc testStudioCase) {
 	}
 
 	// if alias provided, then don't find by name
-	onNameQuery := mockImageReader.On("Query", expectedImageFilter, expectedFindFilter)
+	onNameQuery := mockImageReader.On("Query", image.QueryOptions(expectedImageFilter, expectedFindFilter, false))
 	if aliasName == "" {
-		onNameQuery.Return(images, len(images), nil).Once()
+		onNameQuery.Return(mocks.ImageQueryResult(images, len(images)), nil).Once()
 	} else {
-		onNameQuery.Return(nil, 0, nil).Once()
+		onNameQuery.Return(mocks.ImageQueryResult(nil, 0), nil).Once()
 
 		expectedAliasFilter := &models.ImageFilterType{
 			Organized: &organized,
@@ -216,7 +220,8 @@ func testStudioImages(t *testing.T, tc testStudioCase) {
 			},
 		}
 
-		mockImageReader.On("Query", expectedAliasFilter, expectedFindFilter).Return(images, len(images), nil).Once()
+		mockImageReader.On("Query", image.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
+			Return(mocks.ImageQueryResult(images, len(images)), nil).Once()
 	}
 
 	for i := range matchingPaths {

pkg/autotag/tag.go

@@ -7,42 +7,6 @@ import (
 	"github.com/stashapp/stash/pkg/scene"
 )
 
-func getMatchingTags(path string, tagReader models.TagReader) ([]*models.Tag, error) {
-	words := getPathWords(path)
-	tags, err := tagReader.QueryForAutoTag(words)
-
-	if err != nil {
-		return nil, err
-	}
-
-	var ret []*models.Tag
-	for _, t := range tags {
-		matches := false
-		if nameMatchesPath(t.Name, path) {
-			matches = true
-		}
-
-		if !matches {
-			aliases, err := tagReader.GetAliases(t.ID)
-			if err != nil {
-				return nil, err
-			}
-			for _, alias := range aliases {
-				if nameMatchesPath(alias, path) {
-					matches = true
-					break
-				}
-			}
-		}
-
-		if matches {
-			ret = append(ret, t)
-		}
-	}
-
-	return ret, nil
-}
-
 func getTagTaggers(p *models.Tag, aliases []string) []tagger {
 	ret := []tagger{{
 		ID:   p.ID,

pkg/autotag/tag_test.go

@@ -3,8 +3,10 @@ package autotag
 import (
 	"testing"
 
+	"github.com/stashapp/stash/pkg/image"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/mocks"
+	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -111,11 +113,11 @@ func testTagScenes(t *testing.T, tc testTagCase) {
 	}
 
 	// if alias provided, then don't find by name
-	onNameQuery := mockSceneReader.On("Query", expectedSceneFilter, expectedFindFilter)
+	onNameQuery := mockSceneReader.On("Query", scene.QueryOptions(expectedSceneFilter, expectedFindFilter, false))
 	if aliasName == "" {
-		onNameQuery.Return(scenes, len(scenes), nil).Once()
+		onNameQuery.Return(mocks.SceneQueryResult(scenes, len(scenes)), nil).Once()
 	} else {
-		onNameQuery.Return(nil, 0, nil).Once()
+		onNameQuery.Return(mocks.SceneQueryResult(nil, 0), nil).Once()
 
 		expectedAliasFilter := &models.SceneFilterType{
 			Organized: &organized,
@@ -125,7 +127,8 @@ func testTagScenes(t *testing.T, tc testTagCase) {
 			},
 		}
 
-		mockSceneReader.On("Query", expectedAliasFilter, expectedFindFilter).Return(scenes, len(scenes), nil).Once()
+		mockSceneReader.On("Query", scene.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
+			Return(mocks.SceneQueryResult(scenes, len(scenes)), nil).Once()
 	}
 
 	for i := range matchingPaths {
@@ -198,11 +201,11 @@ func testTagImages(t *testing.T, tc testTagCase) {
 	}
 
 	// if alias provided, then don't find by name
-	onNameQuery := mockImageReader.On("Query", expectedImageFilter, expectedFindFilter)
+	onNameQuery := mockImageReader.On("Query", image.QueryOptions(expectedImageFilter, expectedFindFilter, false))
 	if aliasName == "" {
-		onNameQuery.Return(images, len(images), nil).Once()
+		onNameQuery.Return(mocks.ImageQueryResult(images, len(images)), nil).Once()
 	} else {
-		onNameQuery.Return(nil, 0, nil).Once()
+		onNameQuery.Return(mocks.ImageQueryResult(nil, 0), nil).Once()
 
 		expectedAliasFilter := &models.ImageFilterType{
 			Organized: &organized,
@@ -212,7 +215,8 @@ func testTagImages(t *testing.T, tc testTagCase) {
 			},
 		}
 
-		mockImageReader.On("Query", expectedAliasFilter, expectedFindFilter).Return(images, len(images), nil).Once()
+		mockImageReader.On("Query", image.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
+			Return(mocks.ImageQueryResult(images, len(images)), nil).Once()
 	}
 
 	for i := range matchingPaths {

pkg/autotag/tagger.go

@@ -15,78 +15,12 @@ package autotag
 
 import (
 	"fmt"
-	"path/filepath"
-	"regexp"
-	"strings"
 
 	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
 )
 
-const separatorChars = `.\-_ `
-
-func getPathQueryRegex(name string) string {
-	// escape specific regex characters
-	name = regexp.QuoteMeta(name)
-
-	// handle path separators
-	const separator = `[` + separatorChars + `]`
-
-	ret := strings.Replace(name, " ", separator+"*", -1)
-	ret = `(?:^|_|[^\w\d])` + ret + `(?:$|_|[^\w\d])`
-	return ret
-}
-
-func nameMatchesPath(name, path string) bool {
-	// escape specific regex characters
-	name = regexp.QuoteMeta(name)
-
-	name = strings.ToLower(name)
-	path = strings.ToLower(path)
-
-	// handle path separators
-	const separator = `[` + separatorChars + `]`
-
-	reStr := strings.Replace(name, " ", separator+"*", -1)
-	reStr = `(?:^|_|[^\w\d])` + reStr + `(?:$|_|[^\w\d])`
-
-	re := regexp.MustCompile(reStr)
-	return re.MatchString(path)
-}
-
-func getPathWords(path string) []string {
-	retStr := path
-
-	// remove the extension
-	ext := filepath.Ext(retStr)
-	if ext != "" {
-		retStr = strings.TrimSuffix(retStr, ext)
-	}
-
-	// handle path separators
-	const separator = `(?:_|[^\w\d])+`
-	re := regexp.MustCompile(separator)
-	retStr = re.ReplaceAllString(retStr, " ")
-
-	words := strings.Split(retStr, " ")
-
-	// remove any single letter words
-	var ret []string
-	for _, w := range words {
-		if len(w) > 1 {
-			// #1450 - we need to open up the criteria for matching so that we
-			// can match where path has no space between subject names -
-			// ie name = "foo bar" - path = "foobar"
-			// we post-match afterwards, so we can afford to be a little loose
-			// with the query
-			// just use the first two characters
-			ret = append(ret, w[0:2])
-		}
-	}
-
-	return ret
-}
-
 type tagger struct {
 	ID   int
 	Type string
@@ -105,7 +39,7 @@ func (t *tagger) addLog(otherType, otherName string) {
 }
 
 func (t *tagger) tagPerformers(performerReader models.PerformerReader, addFunc addLinkFunc) error {
-	others, err := getMatchingPerformers(t.Path, performerReader)
+	others, err := match.PathToPerformers(t.Path, performerReader)
 	if err != nil {
 		return err
 	}
@@ -126,7 +60,7 @@ func (t *tagger) tagPerformers(performerReader models.PerformerReader, addFunc a
 }
 
 func (t *tagger) tagStudios(studioReader models.StudioReader, addFunc addLinkFunc) error {
-	others, err := getMatchingStudios(t.Path, studioReader)
+	others, err := match.PathToStudios(t.Path, studioReader)
 	if err != nil {
 		return err
 	}
@@ -149,7 +83,7 @@ func (t *tagger) tagStudios(studioReader models.StudioReader, addFunc addLinkFun
 }
 
 func (t *tagger) tagTags(tagReader models.TagReader, addFunc addLinkFunc) error {
-	others, err := getMatchingTags(t.Path, tagReader)
+	others, err := match.PathToTags(t.Path, tagReader)
 	if err != nil {
 		return err
 	}
@@ -170,7 +104,7 @@ func (t *tagger) tagTags(tagReader models.TagReader, addFunc addLinkFunc) error
 }
 
 func (t *tagger) tagScenes(paths []string, sceneReader models.SceneReader, addFunc addLinkFunc) error {
-	others, err := getMatchingScenes(t.Name, paths, sceneReader)
+	others, err := match.PathToScenes(t.Name, paths, sceneReader)
 	if err != nil {
 		return err
 	}
@@ -191,7 +125,7 @@ func (t *tagger) tagScenes(paths []string, sceneReader models.SceneReader, addFu
 }
 
 func (t *tagger) tagImages(paths []string, imageReader models.ImageReader, addFunc addLinkFunc) error {
-	others, err := getMatchingImages(t.Name, paths, imageReader)
+	others, err := match.PathToImages(t.Name, paths, imageReader)
 	if err != nil {
 		return err
 	}
@@ -212,7 +146,7 @@ func (t *tagger) tagImages(paths []string, imageReader models.ImageReader, addFu
 }
 
 func (t *tagger) tagGalleries(paths []string, galleryReader models.GalleryReader, addFunc addLinkFunc) error {
-	others, err := getMatchingGalleries(t.Name, paths, galleryReader)
+	others, err := match.PathToGalleries(t.Name, paths, galleryReader)
 	if err != nil {
 		return err
 	}

pkg/database/custom_migrations.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"database/sql"
+	"errors"
 	"fmt"
 	"strings"
 
@@ -21,7 +22,7 @@ func createImagesChecksumIndex() error {
 	return WithTxn(func(tx *sqlx.Tx) error {
 		row := tx.QueryRow("SELECT 1 AS found FROM sqlite_master WHERE type = 'index' AND name = 'images_checksum_unique'")
 		err := row.Err()
-		if err != nil && err != sql.ErrNoRows {
+		if err != nil && !errors.Is(err, sql.ErrNoRows) {
 			return err
 		}
 
@@ -55,7 +56,7 @@ func createImagesChecksumIndex() error {
 		}
 
 		err = tx.Select(&result, "SELECT checksum FROM images GROUP BY checksum HAVING COUNT(1) > 1")
-		if err != nil && err != sql.ErrNoRows {
+		if err != nil && !errors.Is(err, sql.ErrNoRows) {
 			logger.Errorf("Unable to determine non-unique image checksums: %s", err)
 			return nil
 		}

pkg/database/database.go

@@ -63,13 +63,13 @@ func Initialize(databasePath string) error {
 	dbPath = databasePath
 
 	if err := getDatabaseSchemaVersion(); err != nil {
-		return fmt.Errorf("error getting database schema version: %s", err.Error())
+		return fmt.Errorf("error getting database schema version: %v", err)
 	}
 
 	if databaseSchemaVersion == 0 {
 		// new database, just run the migrations
 		if err := RunMigrations(); err != nil {
-			return fmt.Errorf("error running initial schema migrations: %s", err.Error())
+			return fmt.Errorf("error running initial schema migrations: %v", err)
 		}
 		// RunMigrations calls Initialise. Just return
 		return nil
@@ -165,7 +165,7 @@ func Backup(db *sqlx.DB, backupPath string) error {
 		var err error
 		db, err = sqlx.Connect(sqlite3Driver, "file:"+dbPath+"?_fk=true")
 		if err != nil {
-			return fmt.Errorf("Open database %s failed:%s", dbPath, err)
+			return fmt.Errorf("open database %s failed: %v", dbPath, err)
 		}
 		defer db.Close()
 	}
@@ -173,7 +173,7 @@ func Backup(db *sqlx.DB, backupPath string) error {
 	logger.Infof("Backing up database into: %s", backupPath)
 	_, err := db.Exec(`VACUUM INTO "` + backupPath + `"`)
 	if err != nil {
-		return fmt.Errorf("vacuum failed: %s", err)
+		return fmt.Errorf("vacuum failed: %v", err)
 	}
 
 	return nil
@@ -298,7 +298,7 @@ func registerCustomDriver() {
 				})
 
 				if err != nil {
-					return fmt.Errorf("error registering natural sort collation: %s", err.Error())
+					return fmt.Errorf("error registering natural sort collation: %v", err)
 				}
 
 				return nil

pkg/database/transaction.go

@@ -22,7 +22,9 @@ func WithTxn(fn func(tx *sqlx.Tx) error) error {
 				logger.Warnf("failure when performing transaction rollback: %v", err)
 			}
 			panic(p)
-		} else if err != nil {
+		}
+
+		if err != nil {
 			// something went wrong, rollback
 			if err := tx.Rollback(); err != nil {
 				logger.Warnf("failure when performing transaction rollback: %v", err)

pkg/dlna/cds.go

@@ -39,6 +39,7 @@ import (
 	"github.com/anacrolix/dms/upnpav"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stashapp/stash/pkg/utils"
 )
 
@@ -437,7 +438,7 @@ func (me *contentDirectoryService) getVideos(sceneFilter *models.SceneFilterType
 			Sort:    &sort,
 		}
 
-		scenes, total, err := r.Scene().Query(sceneFilter, findFilter)
+		scenes, total, err := scene.QueryWithCount(r.Scene(), sceneFilter, findFilter)
 		if err != nil {
 			return err
 		}

pkg/dlna/dms.go

@@ -415,7 +415,7 @@ func (me *Server) serveIcon(w http.ResponseWriter, r *http.Request) {
 	}
 
 	var scene *models.Scene
-	err := me.txnManager.WithReadTxn(context.Background(), func(r models.ReaderRepository) error {
+	err := me.txnManager.WithReadTxn(r.Context(), func(r models.ReaderRepository) error {
 		idInt, err := strconv.Atoi(sceneId)
 		if err != nil {
 			return nil
@@ -434,7 +434,7 @@ func (me *Server) serveIcon(w http.ResponseWriter, r *http.Request) {
 	me.sceneServer.ServeScreenshot(scene, w, r)
 }
 
-func (me *Server) contentDirectoryInitialEvent(urls []*url.URL, sid string) {
+func (me *Server) contentDirectoryInitialEvent(ctx context.Context, urls []*url.URL, sid string) {
 	body := xmlMarshalOrPanic(upnp.PropertySet{
 		Properties: []upnp.Property{
 			{
@@ -465,7 +465,7 @@ func (me *Server) contentDirectoryInitialEvent(urls []*url.URL, sid string) {
 	body = append([]byte(`<?xml version="1.0"?>`+"\n"), body...)
 	for _, _url := range urls {
 		bodyReader := bytes.NewReader(body)
-		req, err := http.NewRequest("NOTIFY", _url.String(), bodyReader)
+		req, err := http.NewRequestWithContext(ctx, "NOTIFY", _url.String(), bodyReader)
 		if err != nil {
 			logger.Errorf("Could not create a request to notify %s: %s", _url.String(), err)
 			continue
@@ -515,7 +515,8 @@ func (me *Server) contentDirectoryEventSubHandler(w http.ResponseWriter, r *http
 	// the spec on eventing but hasn't been completed as I have nothing to
 	// test it with.
 	service := me.services["ContentDirectory"]
-	if r.Method == "SUBSCRIBE" && r.Header.Get("SID") == "" {
+	switch {
+	case r.Method == "SUBSCRIBE" && r.Header.Get("SID") == "":
 		urls := upnp.ParseCallbackURLs(r.Header.Get("CALLBACK"))
 		var timeout int
 		fmt.Sscanf(r.Header.Get("TIMEOUT"), "Second-%d", &timeout)
@@ -526,11 +527,11 @@ func (me *Server) contentDirectoryEventSubHandler(w http.ResponseWriter, r *http
 		w.WriteHeader(http.StatusOK)
 		go func() {
 			time.Sleep(100 * time.Millisecond)
-			me.contentDirectoryInitialEvent(urls, sid)
+			me.contentDirectoryInitialEvent(r.Context(), urls, sid)
 		}()
-	} else if r.Method == "SUBSCRIBE" {
+	case r.Method == "SUBSCRIBE":
 		http.Error(w, "meh", http.StatusPreconditionFailed)
-	} else {
+	default:
 		logger.Debugf("unhandled event method: %s", r.Method)
 	}
 }
@@ -554,7 +555,7 @@ func (me *Server) initMux(mux *http.ServeMux) {
 	mux.HandleFunc(resPath, func(w http.ResponseWriter, r *http.Request) {
 		sceneId := r.URL.Query().Get("scene")
 		var scene *models.Scene
-		err := me.txnManager.WithReadTxn(context.Background(), func(r models.ReaderRepository) error {
+		err := me.txnManager.WithReadTxn(r.Context(), func(r models.ReaderRepository) error {
 			sceneIdInt, err := strconv.Atoi(sceneId)
 			if err != nil {
 				return nil

pkg/dlna/paging.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/scene"
 )
 
 type scenePager struct {
@@ -36,7 +37,7 @@ func (p *scenePager) getPages(r models.ReaderRepository, total int) ([]interface
 		if pages <= 10 || (page-1)%(pages/10) == 0 {
 			thisPage := ((page - 1) * pageSize) + 1
 			findFilter.Page = &thisPage
-			scenes, _, err := r.Scene().Query(p.sceneFilter, findFilter)
+			scenes, err := scene.Query(r.Scene(), p.sceneFilter, findFilter)
 			if err != nil {
 				return nil, err
 			}
@@ -48,7 +49,7 @@ func (p *scenePager) getPages(r models.ReaderRepository, total int) ([]interface
 				sceneTitle = sceneTitle[0:3]
 			}
 
-			title = title + fmt.Sprintf(" (%s...)", sceneTitle)
+			title += fmt.Sprintf(" (%s...)", sceneTitle)
 		}
 
 		objs = append(objs, makeStorageFolder(p.getPageID(page), title, p.parentID))
@@ -67,7 +68,7 @@ func (p *scenePager) getPageVideos(r models.ReaderRepository, page int, host str
 		Sort:    &sort,
 	}
 
-	scenes, _, err := r.Scene().Query(p.sceneFilter, findFilter)
+	scenes, err := scene.Query(r.Scene(), p.sceneFilter, findFilter)
 	if err != nil {
 		return nil, err
 	}

pkg/ffmpeg/downloader.go

@@ -2,6 +2,7 @@ package ffmpeg
 
 import (
 	"archive/zip"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -36,9 +37,9 @@ func GetPaths(paths []string) (string, string) {
 	return ffmpegPath, ffprobePath
 }
 
-func Download(configDirectory string) error {
+func Download(ctx context.Context, configDirectory string) error {
 	for _, url := range getFFMPEGURL() {
-		err := DownloadSingle(configDirectory, url)
+		err := DownloadSingle(ctx, configDirectory, url)
 		if err != nil {
 			return err
 		}
@@ -69,7 +70,7 @@ func (r *progressReader) Read(p []byte) (int, error) {
 	return read, err
 }
 
-func DownloadSingle(configDirectory, url string) error {
+func DownloadSingle(ctx context.Context, configDirectory, url string) error {
 	if url == "" {
 		return fmt.Errorf("no ffmpeg url for this platform")
 	}
@@ -88,7 +89,12 @@ func DownloadSingle(configDirectory, url string) error {
 	logger.Infof("Downloading %s...", url)
 
 	// Make the HTTP request
-	resp, err := http.Get(url)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return err
 	}
@@ -148,9 +154,14 @@ func getFFMPEGURL() []string {
 	case "darwin":
 		urls = []string{"https://evermeet.cx/ffmpeg/ffmpeg-4.3.1.zip", "https://evermeet.cx/ffmpeg/ffprobe-4.3.1.zip"}
 	case "linux":
-		// TODO: get appropriate arch (arm,arm64,amd64) and xz untar from https://johnvansickle.com/ffmpeg/
-		//       or get the ffmpeg,ffprobe zip repackaged ones from  https://ffbinaries.com/downloads
-		urls = []string{""}
+		switch runtime.GOARCH {
+		case "amd64":
+			urls = []string{"https://github.com/ffbinaries/ffbinaries-prebuilt/releases/download/v4.2.1/ffmpeg-4.2.1-linux-64.zip", "https://github.com/ffbinaries/ffbinaries-prebuilt/releases/download/v4.2.1/ffprobe-4.2.1-linux-64.zip"}
+		case "arm":
+			urls = []string{"https://github.com/ffbinaries/ffbinaries-prebuilt/releases/download/v4.2.1/ffmpeg-4.2.1-linux-armhf-32.zip", "https://github.com/ffbinaries/ffbinaries-prebuilt/releases/download/v4.2.1/ffprobe-4.2.1-linux-armhf-32.zip"}
+		case "arm64":
+			urls = []string{"https://github.com/ffbinaries/ffbinaries-prebuilt/releases/download/v4.2.1/ffmpeg-4.2.1-linux-arm-64.zip", "https://github.com/ffbinaries/ffbinaries-prebuilt/releases/download/v4.2.1/ffprobe-4.2.1-linux-arm-64.zip"}
+		}
 	case "windows":
 		urls = []string{"https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.zip"}
 	default:

pkg/ffmpeg/encoder.go

@@ -12,21 +12,13 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 )
 
-type Encoder struct {
-	Path string
-}
+type Encoder string
 
 var (
 	runningEncoders      = make(map[string][]*os.Process)
 	runningEncodersMutex = sync.RWMutex{}
 )
 
-func NewEncoder(ffmpegPath string) Encoder {
-	return Encoder{
-		Path: ffmpegPath,
-	}
-}
-
 func registerRunningEncoder(path string, process *os.Process) {
 	runningEncodersMutex.Lock()
 	processes := runningEncoders[path]
@@ -86,7 +78,7 @@ func KillRunningEncoders(path string) {
 
 // FFmpeg runner with progress output, used for transcodes
 func (e *Encoder) runTranscode(probeResult VideoFile, args []string) (string, error) {
-	cmd := exec.Command(e.Path, args...)
+	cmd := exec.Command(string(*e), args...)
 
 	stderr, err := cmd.StderrPipe()
 	if err != nil {
@@ -141,19 +133,25 @@ func (e *Encoder) runTranscode(probeResult VideoFile, args []string) (string, er
 	return stdoutString, nil
 }
 
-func (e *Encoder) run(probeResult VideoFile, args []string) (string, error) {
-	cmd := exec.Command(e.Path, args...)
+func (e *Encoder) run(sourcePath string, args []string, stdin io.Reader) (string, error) {
+	cmd := exec.Command(string(*e), args...)
 
 	var stdout, stderr bytes.Buffer
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
+	cmd.Stdin = stdin
 
 	if err := cmd.Start(); err != nil {
 		return "", err
 	}
 
-	registerRunningEncoder(probeResult.Path, cmd.Process)
-	err := waitAndDeregister(probeResult.Path, cmd)
+	var err error
+	if sourcePath != "" {
+		registerRunningEncoder(sourcePath, cmd.Process)
+		err = waitAndDeregister(sourcePath, cmd)
+	} else {
+		err = cmd.Wait()
+	}
 
 	if err != nil {
 		// error message should be in the stderr stream

pkg/ffmpeg/encoder_marker.go

@@ -34,7 +34,7 @@ func (e *Encoder) SceneMarkerVideo(probeResult VideoFile, options SceneMarkerOpt
 		"-strict", "-2",
 		options.OutputPath,
 	}
-	_, err := e.run(probeResult, args)
+	_, err := e.run(probeResult.Path, args, nil)
 	return err
 }
 
@@ -55,6 +55,6 @@ func (e *Encoder) SceneMarkerImage(probeResult VideoFile, options SceneMarkerOpt
 		"-an",
 		options.OutputPath,
 	}
-	_, err := e.run(probeResult, args)
+	_, err := e.run(probeResult.Path, args, nil)
 	return err
 }

pkg/ffmpeg/encoder_scene_preview_chunk.go

@@ -85,11 +85,11 @@ func (e *Encoder) ScenePreviewVideoChunk(probeResult VideoFile, options ScenePre
 		"-strict", "-2",
 	}
 
-	args3 := append(args, args2...)
-	args3 = append(args3, argsAudio...)
-	finalArgs := append(args3, options.OutputPath)
+	args = append(args, args2...)
+	args = append(args, argsAudio...)
+	args = append(args, options.OutputPath)
 
-	_, err := e.run(probeResult, finalArgs)
+	_, err := e.run(probeResult.Path, args, nil)
 	return err
 }
 
@@ -102,7 +102,7 @@ func (e *Encoder) ScenePreviewVideoChunkCombine(probeResult VideoFile, concatFil
 		"-c", "copy",
 		outputPath,
 	}
-	_, err := e.run(probeResult, args)
+	_, err := e.run(probeResult.Path, args, nil)
 	return err
 }
 
@@ -122,6 +122,6 @@ func (e *Encoder) ScenePreviewVideoToImage(probeResult VideoFile, width int, vid
 		"-an",
 		outputPath,
 	}
-	_, err := e.run(probeResult, args)
+	_, err := e.run(probeResult.Path, args, nil)
 	return err
 }

pkg/ffmpeg/encoder_screenshot.go

@@ -28,7 +28,7 @@ func (e *Encoder) Screenshot(probeResult VideoFile, options ScreenshotOptions) e
 		"-f", "image2",
 		options.OutputPath,
 	}
-	_, err := e.run(probeResult, args)
+	_, err := e.run(probeResult.Path, args, nil)
 
 	return err
 }

pkg/ffmpeg/encoder_sprite_screenshot.go

@@ -22,7 +22,7 @@ func (e *Encoder) SpriteScreenshot(probeResult VideoFile, options SpriteScreensh
 		"-f", "rawvideo",
 		"-",
 	}
-	data, err := e.run(probeResult, args)
+	data, err := e.run(probeResult.Path, args, nil)
 	if err != nil {
 		return nil, err
 	}

pkg/ffmpeg/encoder_transcode.go

@@ -67,9 +67,9 @@ func (e *Encoder) Transcode(probeResult VideoFile, options TranscodeOptions) {
 	_, _ = e.runTranscode(probeResult, args)
 }
 
-//transcode the video, remove the audio
-//in some videos where the audio codec is not supported by ffmpeg
-//ffmpeg fails if you try to transcode the audio
+// TranscodeVideo transcodes the video, and removes the audio.
+// In some videos where the audio codec is not supported by ffmpeg,
+// ffmpeg fails if you try to transcode the audio
 func (e *Encoder) TranscodeVideo(probeResult VideoFile, options TranscodeOptions) {
 	scale := calculateTranscodeScale(probeResult, options.MaxTranscodeSize)
 	args := []string{
@@ -87,7 +87,7 @@ func (e *Encoder) TranscodeVideo(probeResult VideoFile, options TranscodeOptions
 	_, _ = e.runTranscode(probeResult, args)
 }
 
-//copy the video stream as is, transcode audio
+// TranscodeAudio will copy the video stream as is, and transcode audio.
 func (e *Encoder) TranscodeAudio(probeResult VideoFile, options TranscodeOptions) {
 	args := []string{
 		"-i", probeResult.Path,
@@ -99,7 +99,7 @@ func (e *Encoder) TranscodeAudio(probeResult VideoFile, options TranscodeOptions
 	_, _ = e.runTranscode(probeResult, args)
 }
 
-//copy the video stream as is, drop audio
+// CopyVideo will copy the video stream as is, and drop the audio stream.
 func (e *Encoder) CopyVideo(probeResult VideoFile, options TranscodeOptions) {
 	args := []string{
 		"-i", probeResult.Path,

pkg/ffmpeg/ffprobe.go

@@ -72,8 +72,8 @@ var validAudioForMkv = []AudioCodec{Aac, Mp3, Vorbis, Opus}
 var validAudioForWebm = []AudioCodec{Vorbis, Opus}
 var validAudioForMp4 = []AudioCodec{Aac, Mp3}
 
-//maps user readable container strings to ffprobe's format_name
-//on some formats ffprobe can't differentiate
+// ContainerToFfprobe maps user readable container strings to ffprobe's format_name.
+// On some formats ffprobe can't differentiate
 var ContainerToFfprobe = map[Container]string{
 	Mp4:      Mp4Ffmpeg,
 	M4v:      M4vFfmpeg,
@@ -116,7 +116,7 @@ func IsValidCodec(codecName string, supportedCodecs []string) bool {
 	return false
 }
 
-func IsValidAudio(audio AudioCodec, ValidCodecs []AudioCodec) bool {
+func IsValidAudio(audio AudioCodec, validCodecs []AudioCodec) bool {
 
 	// if audio codec is missing or unsupported by ffmpeg we can't do anything about it
 	// report it as valid so that the file can at least be streamed directly if the video codec is supported
@@ -124,7 +124,7 @@ func IsValidAudio(audio AudioCodec, ValidCodecs []AudioCodec) bool {
 		return true
 	}
 
-	for _, c := range ValidCodecs {
+	for _, c := range validCodecs {
 		if c == audio {
 			return true
 		}
@@ -155,7 +155,8 @@ func IsValidForContainer(format Container, validContainers []Container) bool {
 	return false
 }
 
-//extend stream validation check to take into account container
+// IsValidCombo checks if a codec/container combination is valid.
+// Returns true on validity, false otherwise
 func IsValidCombo(codecName string, format Container, supportedVideoCodecs []string) bool {
 	supportMKV := IsValidCodec(Mkv, supportedVideoCodecs)
 	supportHEVC := IsValidCodec(Hevc, supportedVideoCodecs)
@@ -221,14 +222,13 @@ type VideoFile struct {
 	AudioCodec string
 }
 
+// FFProbe
+type FFProbe string
+
 // Execute exec command and bind result to struct.
-func NewVideoFile(ffprobePath string, videoPath string, stripExt bool) (*VideoFile, error) {
+func (f *FFProbe) NewVideoFile(videoPath string, stripExt bool) (*VideoFile, error) {
 	args := []string{"-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", "-show_error", videoPath}
-	//// Extremely slow on windows for some reason
-	//if runtime.GOOS != "windows" {
-	//	args = append(args, "-count_frames")
-	//}
-	out, err := exec.Command(ffprobePath, args...).Output()
+	out, err := exec.Command(string(*f), args...).Output()
 
 	if err != nil {
 		return nil, fmt.Errorf("FFProbe encountered an error with <%s>.\nError JSON:\n%s\nError: %s", videoPath, string(out), err.Error())
@@ -253,9 +253,6 @@ func parse(filePath string, probeJSON *FFProbeJSON, stripExt bool) (*VideoFile,
 	if result.JSON.Error.Code != 0 {
 		return nil, fmt.Errorf("ffprobe error code %d: %s", result.JSON.Error.Code, result.JSON.Error.String)
 	}
-	//} else if (ffprobeResult.stderr.includes("could not find codec parameters")) {
-	//	throw new Error(`FFProbe [${filePath}] -> Could not find codec parameters`);
-	//} // TODO nil_or_unsupported.(video_stream) && nil_or_unsupported.(audio_stream)
 
 	result.Path = filePath
 	result.Title = probeJSON.Format.Tags.Title

pkg/ffmpeg/image.go

@@ -0,0 +1,40 @@
+package ffmpeg
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+)
+
+var ErrUnsupportedFormat = errors.New("unsupported image format")
+
+func (e *Encoder) ImageThumbnail(image *bytes.Buffer, format *string, maxDimensions int, path string) ([]byte, error) {
+	// ffmpeg spends a long sniffing image format when data is piped through stdio, so we pass the format explicitly instead
+	ffmpegformat := ""
+	if format == nil {
+		return nil, ErrUnsupportedFormat
+	}
+
+	switch *format {
+	case "jpeg":
+		ffmpegformat = "mjpeg"
+	case "png":
+		ffmpegformat = "png_pipe"
+	case "webp":
+		ffmpegformat = "webp_pipe"
+	}
+
+	args := []string{
+		"-f", ffmpegformat,
+		"-i", "-",
+		"-vf", fmt.Sprintf("scale=%v:%v:force_original_aspect_ratio=decrease", maxDimensions, maxDimensions),
+		"-c:v", "mjpeg",
+		"-q:v", "5",
+		"-f", "image2pipe",
+		"-",
+	}
+
+	data, err := e.run(path, args, image)
+
+	return []byte(data), err
+}

pkg/ffmpeg/media_detection.go

@@ -2,8 +2,9 @@ package ffmpeg
 
 import (
 	"bytes"
-	"github.com/stashapp/stash/pkg/logger"
 	"os"
+
+	"github.com/stashapp/stash/pkg/logger"
 )
 
 // detect file format from magic file number
@@ -37,11 +38,12 @@ func containsMatroskaSignature(buf, subType []byte) bool {
 	return buf[index-3] == 0x42 && buf[index-2] == 0x82
 }
 
-//returns container as string ("" on error or no match)
-//implements only mkv or webm as ffprobe can't distinguish between them
-//and not all browsers support mkv
-func MagicContainer(file_path string) Container {
-	file, err := os.Open(file_path)
+// MagicContainer returns the container type of a file path.
+// Returns the zero-value on errors or no-match. Implements mkv or
+// webm only, as ffprobe can't distinguish between them and not all
+// browsers support mkv
+func MagicContainer(filePath string) Container {
+	file, err := os.Open(filePath)
 	if err != nil {
 		logger.Errorf("[magicfile] %v", err)
 		return ""

pkg/ffmpeg/stream.go

@@ -205,7 +205,7 @@ func (e *Encoder) GetTranscodeStream(options TranscodeStreamOptions) (*Stream, e
 
 func (e *Encoder) stream(probeResult VideoFile, options TranscodeStreamOptions) (*Stream, error) {
 	args := options.getStreamArgs()
-	cmd := exec.Command(e.Path, args...)
+	cmd := exec.Command(string(*e), args...)
 	logger.Debugf("Streaming via: %s", strings.Join(cmd.Args, " "))
 
 	stdout, err := cmd.StdoutPipe()

pkg/file/file.go

@@ -0,0 +1,31 @@
+package file
+
+import (
+	"io"
+	"io/fs"
+	"os"
+)
+
+type fsFile struct {
+	path string
+	info fs.FileInfo
+}
+
+func (f *fsFile) Open() (io.ReadCloser, error) {
+	return os.Open(f.path)
+}
+
+func (f *fsFile) Path() string {
+	return f.path
+}
+
+func (f *fsFile) FileInfo() fs.FileInfo {
+	return f.info
+}
+
+func FSFile(path string, info fs.FileInfo) SourceFile {
+	return &fsFile{
+		path: path,
+		info: info,
+	}
+}

pkg/file/hash.go

@@ -0,0 +1,17 @@
+package file
+
+import (
+	"io"
+
+	"github.com/stashapp/stash/pkg/utils"
+)
+
+type FSHasher struct{}
+
+func (h *FSHasher) OSHash(src io.ReadSeeker, size int64) (string, error) {
+	return utils.OSHashFromReader(src, size)
+}
+
+func (h *FSHasher) MD5(src io.Reader) (string, error) {
+	return utils.MD5FromReader(src)
+}

pkg/file/scan.go

@@ -0,0 +1,176 @@
+package file
+
+import (
+	"fmt"
+	"io"
+	"io/fs"
+	"strconv"
+	"time"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+)
+
+type SourceFile interface {
+	Open() (io.ReadCloser, error)
+	Path() string
+	FileInfo() fs.FileInfo
+}
+
+type FileBased interface {
+	File() models.File
+}
+
+type Hasher interface {
+	OSHash(src io.ReadSeeker, size int64) (string, error)
+	MD5(src io.Reader) (string, error)
+}
+
+type Scanned struct {
+	Old *models.File
+	New *models.File
+}
+
+// FileUpdated returns true if both old and new files are present and not equal.
+func (s Scanned) FileUpdated() bool {
+	if s.Old == nil || s.New == nil {
+		return false
+	}
+
+	return !s.Old.Equal(*s.New)
+}
+
+// ContentsChanged returns true if both old and new files are present and the file content is different.
+func (s Scanned) ContentsChanged() bool {
+	if s.Old == nil || s.New == nil {
+		return false
+	}
+
+	if s.Old.Checksum != s.New.Checksum {
+		return true
+	}
+
+	if s.Old.OSHash != s.New.OSHash {
+		return true
+	}
+
+	return false
+}
+
+type Scanner struct {
+	Hasher Hasher
+
+	CalculateMD5    bool
+	CalculateOSHash bool
+}
+
+func (o Scanner) ScanExisting(existing FileBased, file SourceFile) (h *Scanned, err error) {
+	info := file.FileInfo()
+	h = &Scanned{}
+
+	existingFile := existing.File()
+	h.Old = &existingFile
+
+	updatedFile := existingFile
+	h.New = &updatedFile
+
+	// update existing data if needed
+	// truncate to seconds, since we don't store beyond that in the database
+	updatedFile.FileModTime = info.ModTime().Truncate(time.Second)
+	updatedFile.Size = strconv.FormatInt(info.Size(), 10)
+
+	modTimeChanged := !existingFile.FileModTime.Equal(updatedFile.FileModTime)
+
+	// regenerate hash(es) if missing or file mod time changed
+	if _, err = o.generateHashes(&updatedFile, file, modTimeChanged); err != nil {
+		return nil, err
+	}
+
+	// notify of changes as needed
+	// object exists, no further processing required
+	return
+}
+
+func (o Scanner) ScanNew(file SourceFile) (*models.File, error) {
+	info := file.FileInfo()
+	sizeStr := strconv.FormatInt(info.Size(), 10)
+	modTime := info.ModTime()
+	f := models.File{
+		Path:        file.Path(),
+		Size:        sizeStr,
+		FileModTime: modTime,
+	}
+
+	if _, err := o.generateHashes(&f, file, true); err != nil {
+		return nil, err
+	}
+
+	return &f, nil
+}
+
+// generateHashes regenerates and sets the hashes in the provided File.
+// It will not recalculate unless specified.
+func (o Scanner) generateHashes(f *models.File, file SourceFile, regenerate bool) (changed bool, err error) {
+	existing := *f
+
+	var src io.ReadCloser
+	if o.CalculateOSHash && (regenerate || f.OSHash == "") {
+		logger.Infof("Calculating oshash for %s ...", f.Path)
+
+		src, err = file.Open()
+		if err != nil {
+			return false, err
+		}
+		defer src.Close()
+
+		seekSrc, valid := src.(io.ReadSeeker)
+		if !valid {
+			return false, fmt.Errorf("invalid source file type: %s", file.Path())
+		}
+
+		// regenerate hash
+		var oshash string
+		oshash, err = o.Hasher.OSHash(seekSrc, file.FileInfo().Size())
+		if err != nil {
+			return false, fmt.Errorf("error generating oshash for %s: %w", file.Path(), err)
+		}
+
+		f.OSHash = oshash
+
+		// reset reader to start of file
+		_, err = seekSrc.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, fmt.Errorf("error seeking to start of file in %s: %w", file.Path(), err)
+		}
+	}
+
+	// always generate if MD5 is nil
+	// only regenerate MD5 if:
+	// - OSHash was not calculated, or
+	// - existing OSHash is different to generated one
+	// or if it was different to the previous version
+	if o.CalculateMD5 && (f.Checksum == "" || (regenerate && (!o.CalculateOSHash || existing.OSHash != f.OSHash))) {
+		logger.Infof("Calculating checksum for %s...", f.Path)
+
+		if src == nil {
+			src, err = file.Open()
+			if err != nil {
+				return false, err
+			}
+			defer src.Close()
+		}
+
+		// regenerate checksum
+		var checksum string
+		checksum, err = o.Hasher.MD5(src)
+		if err != nil {
+			return
+		}
+
+		f.Checksum = checksum
+	}
+
+	changed = (o.CalculateOSHash && (f.OSHash != existing.OSHash)) || (o.CalculateMD5 && (f.Checksum != existing.Checksum))
+
+	return
+}

pkg/file/zip.go

@@ -0,0 +1,64 @@
+package file
+
+import (
+	"archive/zip"
+	"io"
+	"io/fs"
+	"strings"
+)
+
+const zipSeparator = "\x00"
+
+type zipFile struct {
+	zipPath string
+	zf      *zip.File
+}
+
+func (f *zipFile) Open() (io.ReadCloser, error) {
+	return f.zf.Open()
+}
+
+func (f *zipFile) Path() string {
+	// TODO - fix this
+	return ZipFilename(f.zipPath, f.zf.Name)
+}
+
+func (f *zipFile) FileInfo() fs.FileInfo {
+	return f.zf.FileInfo()
+}
+
+func ZipFile(zipPath string, zf *zip.File) SourceFile {
+	return &zipFile{
+		zipPath: zipPath,
+		zf:      zf,
+	}
+}
+
+func ZipFilename(zipFilename, filenameInZip string) string {
+	return zipFilename + zipSeparator + filenameInZip
+}
+
+// IsZipPath returns true if the path includes the zip separator byte,
+// indicating it is within a zip file.
+func IsZipPath(p string) bool {
+	return strings.Contains(p, zipSeparator)
+}
+
+// ZipPathDisplayName converts an zip path for display. It translates the zip
+// file separator character into '/', since this character is also used for
+// path separators within zip files. It returns the original provided path
+// if it does not contain the zip file separator character.
+func ZipPathDisplayName(path string) string {
+	return strings.ReplaceAll(path, zipSeparator, "/")
+}
+
+func ZipFilePath(path string) (zipFilename, filename string) {
+	nullIndex := strings.Index(path, zipSeparator)
+	if nullIndex != -1 {
+		zipFilename = path[0:nullIndex]
+		filename = path[nullIndex+1:]
+	} else {
+		filename = path
+	}
+	return
+}

pkg/gallery/export_test.go

@@ -25,7 +25,7 @@ const (
 
 const (
 	path      = "path"
-	zip       = true
+	isZip     = true
 	url       = "url"
 	checksum  = "checksum"
 	title     = "title"
@@ -48,7 +48,7 @@ func createFullGallery(id int) models.Gallery {
 	return models.Gallery{
 		ID:       id,
 		Path:     models.NullString(path),
-		Zip:      zip,
+		Zip:      isZip,
 		Title:    models.NullString(title),
 		Checksum: checksum,
 		Date: models.SQLiteDate{
@@ -72,7 +72,7 @@ func createFullJSONGallery() *jsonschema.Gallery {
 	return &jsonschema.Gallery{
 		Title:     title,
 		Path:      path,
-		Zip:       zip,
+		Zip:       isZip,
 		Checksum:  checksum,
 		Date:      date,
 		Details:   details,
@@ -107,11 +107,12 @@ func TestToJSON(t *testing.T) {
 		gallery := s.input
 		json, err := ToBasicJSON(&gallery)
 
-		if !s.err && err != nil {
+		switch {
+		case !s.err && err != nil:
 			t.Errorf("[%d] unexpected error: %s", i, err.Error())
-		} else if s.err && err == nil {
+		case s.err && err == nil:
 			t.Errorf("[%d] expected error not returned", i)
-		} else {
+		default:
 			assert.Equal(t, s.expected, json, "[%d]", i)
 		}
 	}
@@ -162,11 +163,12 @@ func TestGetStudioName(t *testing.T) {
 		gallery := s.input
 		json, err := GetStudioName(mockStudioReader, &gallery)
 
-		if !s.err && err != nil {
+		switch {
+		case !s.err && err != nil:
 			t.Errorf("[%d] unexpected error: %s", i, err.Error())
-		} else if s.err && err == nil {
+		case s.err && err == nil:
 			t.Errorf("[%d] expected error not returned", i)
-		} else {
+		default:
 			assert.Equal(t, s.expected, json, "[%d]", i)
 		}
 	}

pkg/gallery/import.go

@@ -78,7 +78,7 @@ func (i *Importer) populateStudio() error {
 	if i.Input.Studio != "" {
 		studio, err := i.StudioWriter.FindByName(i.Input.Studio, false)
 		if err != nil {
-			return fmt.Errorf("error finding studio by name: %s", err.Error())
+			return fmt.Errorf("error finding studio by name: %v", err)
 		}
 
 		if studio == nil {
@@ -147,7 +147,7 @@ func (i *Importer) populatePerformers() error {
 			if i.MissingRefBehaviour == models.ImportMissingRefEnumCreate {
 				createdPerformers, err := i.createPerformers(missingPerformers)
 				if err != nil {
-					return fmt.Errorf("error creating gallery performers: %s", err.Error())
+					return fmt.Errorf("error creating gallery performers: %v", err)
 				}
 
 				performers = append(performers, createdPerformers...)
@@ -203,7 +203,7 @@ func (i *Importer) populateTags() error {
 			if i.MissingRefBehaviour == models.ImportMissingRefEnumCreate {
 				createdTags, err := i.createTags(missingTags)
 				if err != nil {
-					return fmt.Errorf("error creating gallery tags: %s", err.Error())
+					return fmt.Errorf("error creating gallery tags: %v", err)
 				}
 
 				tags = append(tags, createdTags...)
@@ -242,7 +242,7 @@ func (i *Importer) PostImport(id int) error {
 		}
 
 		if err := i.ReaderWriter.UpdatePerformers(id, performerIDs); err != nil {
-			return fmt.Errorf("failed to associate performers: %s", err.Error())
+			return fmt.Errorf("failed to associate performers: %v", err)
 		}
 	}
 
@@ -252,7 +252,7 @@ func (i *Importer) PostImport(id int) error {
 			tagIDs = append(tagIDs, t.ID)
 		}
 		if err := i.ReaderWriter.UpdateTags(id, tagIDs); err != nil {
-			return fmt.Errorf("failed to associate tags: %s", err.Error())
+			return fmt.Errorf("failed to associate tags: %v", err)
 		}
 	}
 
@@ -280,7 +280,7 @@ func (i *Importer) FindExistingID() (*int, error) {
 func (i *Importer) Create() (*int, error) {
 	created, err := i.ReaderWriter.Create(i.gallery)
 	if err != nil {
-		return nil, fmt.Errorf("error creating gallery: %s", err.Error())
+		return nil, fmt.Errorf("error creating gallery: %v", err)
 	}
 
 	id := created.ID
@@ -292,7 +292,7 @@ func (i *Importer) Update(id int) error {
 	gallery.ID = id
 	_, err := i.ReaderWriter.Update(gallery)
 	if err != nil {
-		return fmt.Errorf("error updating existing gallery: %s", err.Error())
+		return fmt.Errorf("error updating existing gallery: %v", err)
 	}
 
 	return nil

pkg/gallery/scan.go

@@ -0,0 +1,225 @@
+package gallery
+
+import (
+	"archive/zip"
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/stashapp/stash/pkg/file"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/manager/paths"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/plugin"
+	"github.com/stashapp/stash/pkg/utils"
+)
+
+const mutexType = "gallery"
+
+type Scanner struct {
+	file.Scanner
+
+	ImageExtensions    []string
+	StripFileExtension bool
+	Ctx                context.Context
+	CaseSensitiveFs    bool
+	TxnManager         models.TransactionManager
+	Paths              *paths.Paths
+	PluginCache        *plugin.Cache
+	MutexManager       *utils.MutexManager
+}
+
+func FileScanner(hasher file.Hasher) file.Scanner {
+	return file.Scanner{
+		Hasher:       hasher,
+		CalculateMD5: true,
+	}
+}
+
+func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFile) (retGallery *models.Gallery, scanImages bool, err error) {
+	scanned, err := scanner.Scanner.ScanExisting(existing, file)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// we don't currently store sizes for gallery files
+	// clear the file size so that we don't incorrectly detect a
+	// change
+	scanned.New.Size = ""
+
+	retGallery = existing.(*models.Gallery)
+
+	path := scanned.New.Path
+
+	changed := false
+
+	if scanned.ContentsChanged() {
+		retGallery.SetFile(*scanned.New)
+		changed = true
+	} else if scanned.FileUpdated() {
+		logger.Infof("Updated gallery file %s", path)
+
+		retGallery.SetFile(*scanned.New)
+		changed = true
+	}
+
+	if changed {
+		scanImages = true
+		logger.Infof("%s has been updated: rescanning", path)
+
+		retGallery.UpdatedAt = models.SQLiteTimestamp{Timestamp: time.Now()}
+
+		// we are operating on a checksum now, so grab a mutex on the checksum
+		done := make(chan struct{})
+		scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done)
+
+		if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error {
+			// free the mutex once transaction is complete
+			defer close(done)
+
+			// ensure no clashes of hashes
+			if scanned.New.Checksum != "" && scanned.Old.Checksum != scanned.New.Checksum {
+				dupe, _ := r.Gallery().FindByChecksum(retGallery.Checksum)
+				if dupe != nil {
+					return fmt.Errorf("MD5 for file %s is the same as that of %s", path, dupe.Path.String)
+				}
+			}
+
+			retGallery, err = r.Gallery().Update(*retGallery)
+			return err
+		}); err != nil {
+			return nil, false, err
+		}
+
+		scanner.PluginCache.ExecutePostHooks(scanner.Ctx, retGallery.ID, plugin.GalleryUpdatePost, nil, nil)
+	}
+
+	return
+}
+
+func (scanner *Scanner) ScanNew(file file.SourceFile) (retGallery *models.Gallery, scanImages bool, err error) {
+	scanned, err := scanner.Scanner.ScanNew(file)
+	if err != nil {
+		return nil, false, err
+	}
+
+	path := file.Path()
+	checksum := scanned.Checksum
+	isNewGallery := false
+	isUpdatedGallery := false
+	var g *models.Gallery
+
+	// grab a mutex on the checksum
+	done := make(chan struct{})
+	scanner.MutexManager.Claim(mutexType, checksum, done)
+	defer close(done)
+
+	if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error {
+		qb := r.Gallery()
+
+		g, _ = qb.FindByChecksum(checksum)
+		if g != nil {
+			exists, _ := utils.FileExists(g.Path.String)
+			if !scanner.CaseSensitiveFs {
+				// #1426 - if file exists but is a case-insensitive match for the
+				// original filename, then treat it as a move
+				if exists && strings.EqualFold(path, g.Path.String) {
+					exists = false
+				}
+			}
+
+			if exists {
+				logger.Infof("%s already exists.  Duplicate of %s ", path, g.Path.String)
+			} else {
+				logger.Infof("%s already exists.  Updating path...", path)
+				g.Path = sql.NullString{
+					String: path,
+					Valid:  true,
+				}
+				g, err = qb.Update(*g)
+				if err != nil {
+					return err
+				}
+
+				isUpdatedGallery = true
+			}
+		} else if scanner.hasImages(path) { // don't create gallery if it has no images
+			currentTime := time.Now()
+
+			g = &models.Gallery{
+				Zip: true,
+				Title: sql.NullString{
+					String: utils.GetNameFromPath(path, scanner.StripFileExtension),
+					Valid:  true,
+				},
+				CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime},
+				UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime},
+			}
+
+			g.SetFile(*scanned)
+
+			// only warn when creating the gallery
+			ok, err := utils.IsZipFileUncompressed(path)
+			if err == nil && !ok {
+				logger.Warnf("%s is using above store (0) level compression.", path)
+			}
+
+			logger.Infof("%s doesn't exist.  Creating new item...", path)
+			g, err = qb.Create(*g)
+			if err != nil {
+				return err
+			}
+
+			scanImages = true
+			isNewGallery = true
+		}
+
+		return nil
+	}); err != nil {
+		return nil, false, err
+	}
+
+	if isNewGallery {
+		scanner.PluginCache.ExecutePostHooks(scanner.Ctx, g.ID, plugin.GalleryCreatePost, nil, nil)
+	} else if isUpdatedGallery {
+		scanner.PluginCache.ExecutePostHooks(scanner.Ctx, g.ID, plugin.GalleryUpdatePost, nil, nil)
+	}
+
+	scanImages = isNewGallery
+	retGallery = g
+
+	return
+}
+
+func (scanner *Scanner) isImage(pathname string) bool {
+	return utils.MatchExtension(pathname, scanner.ImageExtensions)
+}
+
+func (scanner *Scanner) hasImages(path string) bool {
+	readCloser, err := zip.OpenReader(path)
+	if err != nil {
+		logger.Warnf("Error while walking gallery zip: %v", err)
+		return false
+	}
+	defer readCloser.Close()
+
+	for _, file := range readCloser.File {
+		if file.FileInfo().IsDir() {
+			continue
+		}
+
+		if strings.Contains(file.Name, "__MACOSX") {
+			continue
+		}
+
+		if !scanner.isImage(file.Name) {
+			continue
+		}
+
+		return true
+	}
+
+	return false
+}

pkg/identify/identify.go

@@ -0,0 +1,274 @@
+package identify
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/scene"
+	"github.com/stashapp/stash/pkg/utils"
+)
+
+type SceneScraper interface {
+	ScrapeScene(sceneID int) (*models.ScrapedScene, error)
+}
+
+type SceneUpdatePostHookExecutor interface {
+	ExecuteSceneUpdatePostHooks(ctx context.Context, input models.SceneUpdateInput, inputFields []string)
+}
+
+type ScraperSource struct {
+	Name       string
+	Options    *models.IdentifyMetadataOptionsInput
+	Scraper    SceneScraper
+	RemoteSite string
+}
+
+type SceneIdentifier struct {
+	DefaultOptions              *models.IdentifyMetadataOptionsInput
+	Sources                     []ScraperSource
+	ScreenshotSetter            scene.ScreenshotSetter
+	SceneUpdatePostHookExecutor SceneUpdatePostHookExecutor
+}
+
+func (t *SceneIdentifier) Identify(ctx context.Context, txnManager models.TransactionManager, scene *models.Scene) error {
+	result, err := t.scrapeScene(scene)
+	if err != nil {
+		return err
+	}
+
+	if result == nil {
+		logger.Infof("Unable to identify %s", scene.Path)
+		return nil
+	}
+
+	// results were found, modify the scene
+	if err := t.modifyScene(ctx, txnManager, scene, result); err != nil {
+		return fmt.Errorf("error modifying scene: %v", err)
+	}
+
+	return nil
+}
+
+type scrapeResult struct {
+	result *models.ScrapedScene
+	source ScraperSource
+}
+
+func (t *SceneIdentifier) scrapeScene(scene *models.Scene) (*scrapeResult, error) {
+	// iterate through the input sources
+	for _, source := range t.Sources {
+		// scrape using the source
+		scraped, err := source.Scraper.ScrapeScene(scene.ID)
+		if err != nil {
+			return nil, fmt.Errorf("error scraping from %v: %v", source.Scraper, err)
+		}
+
+		// if results were found then return
+		if scraped != nil {
+			return &scrapeResult{
+				result: scraped,
+				source: source,
+			}, nil
+		}
+	}
+
+	return nil, nil
+}
+
+func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene, result *scrapeResult, repo models.Repository) (*scene.UpdateSet, error) {
+	ret := &scene.UpdateSet{
+		ID: s.ID,
+	}
+
+	options := []models.IdentifyMetadataOptionsInput{}
+	if result.source.Options != nil {
+		options = append(options, *result.source.Options)
+	}
+	if t.DefaultOptions != nil {
+		options = append(options, *t.DefaultOptions)
+	}
+
+	fieldOptions := getFieldOptions(options)
+
+	setOrganized := false
+	for _, o := range options {
+		if o.SetOrganized != nil {
+			setOrganized = *o.SetOrganized
+			break
+		}
+	}
+
+	scraped := result.result
+
+	rel := sceneRelationships{
+		repo:         repo,
+		scene:        s,
+		result:       result,
+		fieldOptions: fieldOptions,
+	}
+
+	ret.Partial = getScenePartial(s, scraped, fieldOptions, setOrganized)
+
+	studioID, err := rel.studio()
+	if err != nil {
+		return nil, fmt.Errorf("error getting studio: %w", err)
+	}
+
+	if studioID != nil {
+		ret.Partial.StudioID = &sql.NullInt64{
+			Int64: *studioID,
+			Valid: true,
+		}
+	}
+
+	ignoreMale := false
+	for _, o := range options {
+		if o.IncludeMalePerformers != nil {
+			ignoreMale = !*o.IncludeMalePerformers
+			break
+		}
+	}
+
+	ret.PerformerIDs, err = rel.performers(ignoreMale)
+	if err != nil {
+		return nil, err
+	}
+
+	ret.TagIDs, err = rel.tags()
+	if err != nil {
+		return nil, err
+	}
+
+	ret.StashIDs, err = rel.stashIDs()
+	if err != nil {
+		return nil, err
+	}
+
+	setCoverImage := false
+	for _, o := range options {
+		if o.SetCoverImage != nil {
+			setCoverImage = *o.SetCoverImage
+			break
+		}
+	}
+
+	if setCoverImage {
+		ret.CoverImage, err = rel.cover(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return ret, nil
+}
+
+func (t *SceneIdentifier) modifyScene(ctx context.Context, txnManager models.TransactionManager, s *models.Scene, result *scrapeResult) error {
+	var updater *scene.UpdateSet
+	if err := txnManager.WithTxn(ctx, func(repo models.Repository) error {
+		var err error
+		updater, err = t.getSceneUpdater(ctx, s, result, repo)
+		if err != nil {
+			return err
+		}
+
+		// don't update anything if nothing was set
+		if updater.IsEmpty() {
+			logger.Infof("Nothing to set for %s", s.Path)
+			return nil
+		}
+
+		_, err = updater.Update(repo.Scene(), t.ScreenshotSetter)
+		if err != nil {
+			return fmt.Errorf("error updating scene: %w", err)
+		}
+
+		as := ""
+		title := updater.Partial.Title
+		if title != nil {
+			as = fmt.Sprintf(" as %s", title.String)
+		}
+		logger.Infof("Successfully identified %s%s using %s", s.Path, as, result.source.Name)
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	// fire post-update hooks
+	if !updater.IsEmpty() {
+		updateInput := updater.UpdateInput()
+		fields := utils.NotNilFields(updateInput, "json")
+		t.SceneUpdatePostHookExecutor.ExecuteSceneUpdatePostHooks(ctx, updateInput, fields)
+	}
+
+	return nil
+}
+
+func getFieldOptions(options []models.IdentifyMetadataOptionsInput) map[string]*models.IdentifyFieldOptionsInput {
+	// prefer source-specific field strategies, then the defaults
+	ret := make(map[string]*models.IdentifyFieldOptionsInput)
+	for _, oo := range options {
+		for _, f := range oo.FieldOptions {
+			if _, found := ret[f.Field]; !found {
+				ret[f.Field] = f
+			}
+		}
+	}
+
+	return ret
+}
+
+func getScenePartial(scene *models.Scene, scraped *models.ScrapedScene, fieldOptions map[string]*models.IdentifyFieldOptionsInput, setOrganized bool) models.ScenePartial {
+	partial := models.ScenePartial{
+		ID: scene.ID,
+	}
+
+	if scraped.Title != nil && scene.Title.String != *scraped.Title {
+		if shouldSetSingleValueField(fieldOptions["title"], scene.Title.String != "") {
+			partial.Title = models.NullStringPtr(*scraped.Title)
+		}
+	}
+	if scraped.Date != nil && scene.Date.String != *scraped.Date {
+		if shouldSetSingleValueField(fieldOptions["date"], scene.Date.Valid) {
+			partial.Date = &models.SQLiteDate{
+				String: *scraped.Date,
+				Valid:  true,
+			}
+		}
+	}
+	if scraped.Details != nil && scene.Details.String != *scraped.Details {
+		if shouldSetSingleValueField(fieldOptions["details"], scene.Details.String != "") {
+			partial.Details = models.NullStringPtr(*scraped.Details)
+		}
+	}
+	if scraped.URL != nil && scene.URL.String != *scraped.URL {
+		if shouldSetSingleValueField(fieldOptions["url"], scene.URL.String != "") {
+			partial.URL = models.NullStringPtr(*scraped.URL)
+		}
+	}
+
+	if setOrganized && !scene.Organized {
+		// just reuse the boolean since we know it's true
+		partial.Organized = &setOrganized
+	}
+
+	return partial
+}
+
+func shouldSetSingleValueField(strategy *models.IdentifyFieldOptionsInput, hasExistingValue bool) bool {
+	// if unset then default to MERGE
+	fs := models.IdentifyFieldStrategyMerge
+
+	if strategy != nil && strategy.Strategy.IsValid() {
+		fs = strategy.Strategy
+	}
+
+	if fs == models.IdentifyFieldStrategyIgnore {
+		return false
+	}
+
+	return !hasExistingValue || fs == models.IdentifyFieldStrategyOverwrite
+}

pkg/identify/identify_test.go

@@ -0,0 +1,502 @@
+package identify
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"testing"
+
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/models/mocks"
+	"github.com/stashapp/stash/pkg/utils"
+	"github.com/stretchr/testify/mock"
+)
+
+type mockSceneScraper struct {
+	errIDs  []int
+	results map[int]*models.ScrapedScene
+}
+
+func (s mockSceneScraper) ScrapeScene(sceneID int) (*models.ScrapedScene, error) {
+	if utils.IntInclude(s.errIDs, sceneID) {
+		return nil, errors.New("scrape scene error")
+	}
+	return s.results[sceneID], nil
+}
+
+type mockHookExecutor struct {
+}
+
+func (s mockHookExecutor) ExecuteSceneUpdatePostHooks(ctx context.Context, input models.SceneUpdateInput, inputFields []string) {
+}
+
+func TestSceneIdentifier_Identify(t *testing.T) {
+	const (
+		errID1 = iota
+		errID2
+		missingID
+		found1ID
+		found2ID
+		errUpdateID
+	)
+
+	var scrapedTitle = "scrapedTitle"
+
+	defaultOptions := &models.IdentifyMetadataOptionsInput{}
+	sources := []ScraperSource{
+		{
+			Scraper: mockSceneScraper{
+				errIDs: []int{errID1},
+				results: map[int]*models.ScrapedScene{
+					found1ID: {
+						Title: &scrapedTitle,
+					},
+				},
+			},
+		},
+		{
+			Scraper: mockSceneScraper{
+				errIDs: []int{errID2},
+				results: map[int]*models.ScrapedScene{
+					found2ID: {
+						Title: &scrapedTitle,
+					},
+					errUpdateID: {
+						Title: &scrapedTitle,
+					},
+				},
+			},
+		},
+	}
+
+	repo := mocks.NewTransactionManager()
+	repo.Scene().(*mocks.SceneReaderWriter).On("Update", mock.MatchedBy(func(partial models.ScenePartial) bool {
+		return partial.ID != errUpdateID
+	})).Return(nil, nil)
+	repo.Scene().(*mocks.SceneReaderWriter).On("Update", mock.MatchedBy(func(partial models.ScenePartial) bool {
+		return partial.ID == errUpdateID
+	})).Return(nil, errors.New("update error"))
+
+	tests := []struct {
+		name    string
+		sceneID int
+		wantErr bool
+	}{
+		{
+			"error scraping",
+			errID1,
+			true,
+		},
+		{
+			"error scraping from second",
+			errID2,
+			true,
+		},
+		{
+			"found in first scraper",
+			found1ID,
+			false,
+		},
+		{
+			"found in second scraper",
+			found2ID,
+			false,
+		},
+		{
+			"not found",
+			missingID,
+			false,
+		},
+		{
+			"error modifying",
+			errUpdateID,
+			true,
+		},
+	}
+
+	identifier := SceneIdentifier{
+		DefaultOptions:              defaultOptions,
+		Sources:                     sources,
+		SceneUpdatePostHookExecutor: mockHookExecutor{},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scene := &models.Scene{
+				ID: tt.sceneID,
+			}
+			if err := identifier.Identify(context.TODO(), repo, scene); (err != nil) != tt.wantErr {
+				t.Errorf("SceneIdentifier.Identify() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestSceneIdentifier_modifyScene(t *testing.T) {
+	repo := mocks.NewTransactionManager()
+	tr := &SceneIdentifier{}
+
+	type args struct {
+		scene  *models.Scene
+		result *scrapeResult
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			"empty update",
+			args{
+				&models.Scene{},
+				&scrapeResult{
+					result: &models.ScrapedScene{},
+				},
+			},
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tr.modifyScene(context.TODO(), repo, tt.args.scene, tt.args.result); (err != nil) != tt.wantErr {
+				t.Errorf("SceneIdentifier.modifyScene() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func Test_getFieldOptions(t *testing.T) {
+	const (
+		inFirst  = "inFirst"
+		inSecond = "inSecond"
+		inBoth   = "inBoth"
+	)
+
+	type args struct {
+		options []models.IdentifyMetadataOptionsInput
+	}
+	tests := []struct {
+		name string
+		args args
+		want map[string]*models.IdentifyFieldOptionsInput
+	}{
+		{
+			"simple",
+			args{
+				[]models.IdentifyMetadataOptionsInput{
+					{
+						FieldOptions: []*models.IdentifyFieldOptionsInput{
+							{
+								Field:    inFirst,
+								Strategy: models.IdentifyFieldStrategyIgnore,
+							},
+							{
+								Field:    inBoth,
+								Strategy: models.IdentifyFieldStrategyIgnore,
+							},
+						},
+					},
+					{
+						FieldOptions: []*models.IdentifyFieldOptionsInput{
+							{
+								Field:    inSecond,
+								Strategy: models.IdentifyFieldStrategyMerge,
+							},
+							{
+								Field:    inBoth,
+								Strategy: models.IdentifyFieldStrategyMerge,
+							},
+						},
+					},
+				},
+			},
+			map[string]*models.IdentifyFieldOptionsInput{
+				inFirst: {
+					Field:    inFirst,
+					Strategy: models.IdentifyFieldStrategyIgnore,
+				},
+				inSecond: {
+					Field:    inSecond,
+					Strategy: models.IdentifyFieldStrategyMerge,
+				},
+				inBoth: {
+					Field:    inBoth,
+					Strategy: models.IdentifyFieldStrategyIgnore,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getFieldOptions(tt.args.options); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getFieldOptions() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_getScenePartial(t *testing.T) {
+	var (
+		originalTitle   = "originalTitle"
+		originalDate    = "originalDate"
+		originalDetails = "originalDetails"
+		originalURL     = "originalURL"
+	)
+
+	var (
+		scrapedTitle   = "scrapedTitle"
+		scrapedDate    = "scrapedDate"
+		scrapedDetails = "scrapedDetails"
+		scrapedURL     = "scrapedURL"
+	)
+
+	originalScene := &models.Scene{
+		Title: models.NullString(originalTitle),
+		Date: models.SQLiteDate{
+			String: originalDate,
+			Valid:  true,
+		},
+		Details: models.NullString(originalDetails),
+		URL:     models.NullString(originalURL),
+	}
+
+	organisedScene := *originalScene
+	organisedScene.Organized = true
+
+	emptyScene := &models.Scene{}
+
+	postPartial := models.ScenePartial{
+		Title: models.NullStringPtr(scrapedTitle),
+		Date: &models.SQLiteDate{
+			String: scrapedDate,
+			Valid:  true,
+		},
+		Details: models.NullStringPtr(scrapedDetails),
+		URL:     models.NullStringPtr(scrapedURL),
+	}
+
+	scrapedScene := &models.ScrapedScene{
+		Title:   &scrapedTitle,
+		Date:    &scrapedDate,
+		Details: &scrapedDetails,
+		URL:     &scrapedURL,
+	}
+
+	scrapedUnchangedScene := &models.ScrapedScene{
+		Title:   &originalTitle,
+		Date:    &originalDate,
+		Details: &originalDetails,
+		URL:     &originalURL,
+	}
+
+	makeFieldOptions := func(input *models.IdentifyFieldOptionsInput) map[string]*models.IdentifyFieldOptionsInput {
+		return map[string]*models.IdentifyFieldOptionsInput{
+			"title":   input,
+			"date":    input,
+			"details": input,
+			"url":     input,
+		}
+	}
+
+	overwriteAll := makeFieldOptions(&models.IdentifyFieldOptionsInput{
+		Strategy: models.IdentifyFieldStrategyOverwrite,
+	})
+	ignoreAll := makeFieldOptions(&models.IdentifyFieldOptionsInput{
+		Strategy: models.IdentifyFieldStrategyIgnore,
+	})
+	mergeAll := makeFieldOptions(&models.IdentifyFieldOptionsInput{
+		Strategy: models.IdentifyFieldStrategyMerge,
+	})
+
+	setOrganised := true
+
+	type args struct {
+		scene        *models.Scene
+		scraped      *models.ScrapedScene
+		fieldOptions map[string]*models.IdentifyFieldOptionsInput
+		setOrganized bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want models.ScenePartial
+	}{
+		{
+			"overwrite all",
+			args{
+				originalScene,
+				scrapedScene,
+				overwriteAll,
+				false,
+			},
+			postPartial,
+		},
+		{
+			"ignore all",
+			args{
+				originalScene,
+				scrapedScene,
+				ignoreAll,
+				false,
+			},
+			models.ScenePartial{},
+		},
+		{
+			"merge (existing values)",
+			args{
+				originalScene,
+				scrapedScene,
+				mergeAll,
+				false,
+			},
+			models.ScenePartial{},
+		},
+		{
+			"merge (empty values)",
+			args{
+				emptyScene,
+				scrapedScene,
+				mergeAll,
+				false,
+			},
+			postPartial,
+		},
+		{
+			"unchanged",
+			args{
+				originalScene,
+				scrapedUnchangedScene,
+				overwriteAll,
+				false,
+			},
+			models.ScenePartial{},
+		},
+		{
+			"set organized",
+			args{
+				originalScene,
+				scrapedUnchangedScene,
+				overwriteAll,
+				true,
+			},
+			models.ScenePartial{
+				Organized: &setOrganised,
+			},
+		},
+		{
+			"set organized unchanged",
+			args{
+				&organisedScene,
+				scrapedUnchangedScene,
+				overwriteAll,
+				true,
+			},
+			models.ScenePartial{},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getScenePartial(tt.args.scene, tt.args.scraped, tt.args.fieldOptions, tt.args.setOrganized); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getScenePartial() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_shouldSetSingleValueField(t *testing.T) {
+	const invalid = "invalid"
+
+	type args struct {
+		strategy         *models.IdentifyFieldOptionsInput
+		hasExistingValue bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			"ignore",
+			args{
+				&models.IdentifyFieldOptionsInput{
+					Strategy: models.IdentifyFieldStrategyIgnore,
+				},
+				false,
+			},
+			false,
+		},
+		{
+			"merge existing",
+			args{
+				&models.IdentifyFieldOptionsInput{
+					Strategy: models.IdentifyFieldStrategyMerge,
+				},
+				true,
+			},
+			false,
+		},
+		{
+			"merge absent",
+			args{
+				&models.IdentifyFieldOptionsInput{
+					Strategy: models.IdentifyFieldStrategyMerge,
+				},
+				false,
+			},
+			true,
+		},
+		{
+			"overwrite",
+			args{
+				&models.IdentifyFieldOptionsInput{
+					Strategy: models.IdentifyFieldStrategyOverwrite,
+				},
+				true,
+			},
+			true,
+		},
+		{
+			"nil (merge) existing",
+			args{
+				&models.IdentifyFieldOptionsInput{},
+				true,
+			},
+			false,
+		},
+		{
+			"nil (merge) absent",
+			args{
+				&models.IdentifyFieldOptionsInput{},
+				false,
+			},
+			true,
+		},
+		{
+			"invalid (merge) existing",
+			args{
+				&models.IdentifyFieldOptionsInput{
+					Strategy: invalid,
+				},
+				true,
+			},
+			false,
+		},
+		{
+			"invalid (merge) absent",
+			args{
+				&models.IdentifyFieldOptionsInput{
+					Strategy: invalid,
+				},
+				false,
+			},
+			true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := shouldSetSingleValueField(tt.args.strategy, tt.args.hasExistingValue); got != tt.want {
+				t.Errorf("shouldSetSingleValueField() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/identify/performer.go

@@ -0,0 +1,108 @@
+package identify
+
+import (
+	"database/sql"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/utils"
+)
+
+func getPerformerID(endpoint string, r models.Repository, p *models.ScrapedPerformer, createMissing bool) (*int, error) {
+	if p.StoredID != nil {
+		// existing performer, just add it
+		performerID, err := strconv.Atoi(*p.StoredID)
+		if err != nil {
+			return nil, fmt.Errorf("error converting performer ID %s: %w", *p.StoredID, err)
+		}
+
+		return &performerID, nil
+	} else if createMissing && p.Name != nil { // name is mandatory
+		return createMissingPerformer(endpoint, r, p)
+	}
+
+	return nil, nil
+}
+
+func createMissingPerformer(endpoint string, r models.Repository, p *models.ScrapedPerformer) (*int, error) {
+	created, err := r.Performer().Create(scrapedToPerformerInput(p))
+	if err != nil {
+		return nil, fmt.Errorf("error creating performer: %w", err)
+	}
+
+	if endpoint != "" && p.RemoteSiteID != nil {
+		if err := r.Performer().UpdateStashIDs(created.ID, []models.StashID{
+			{
+				Endpoint: endpoint,
+				StashID:  *p.RemoteSiteID,
+			},
+		}); err != nil {
+			return nil, fmt.Errorf("error setting performer stash id: %w", err)
+		}
+	}
+
+	return &created.ID, nil
+}
+
+func scrapedToPerformerInput(performer *models.ScrapedPerformer) models.Performer {
+	currentTime := time.Now()
+	ret := models.Performer{
+		Name:      sql.NullString{String: *performer.Name, Valid: true},
+		Checksum:  utils.MD5FromString(*performer.Name),
+		CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime},
+		UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime},
+		Favorite:  sql.NullBool{Bool: false, Valid: true},
+	}
+	if performer.Birthdate != nil {
+		ret.Birthdate = models.SQLiteDate{String: *performer.Birthdate, Valid: true}
+	}
+	if performer.DeathDate != nil {
+		ret.DeathDate = models.SQLiteDate{String: *performer.DeathDate, Valid: true}
+	}
+	if performer.Gender != nil {
+		ret.Gender = sql.NullString{String: *performer.Gender, Valid: true}
+	}
+	if performer.Ethnicity != nil {
+		ret.Ethnicity = sql.NullString{String: *performer.Ethnicity, Valid: true}
+	}
+	if performer.Country != nil {
+		ret.Country = sql.NullString{String: *performer.Country, Valid: true}
+	}
+	if performer.EyeColor != nil {
+		ret.EyeColor = sql.NullString{String: *performer.EyeColor, Valid: true}
+	}
+	if performer.HairColor != nil {
+		ret.HairColor = sql.NullString{String: *performer.HairColor, Valid: true}
+	}
+	if performer.Height != nil {
+		ret.Height = sql.NullString{String: *performer.Height, Valid: true}
+	}
+	if performer.Measurements != nil {
+		ret.Measurements = sql.NullString{String: *performer.Measurements, Valid: true}
+	}
+	if performer.FakeTits != nil {
+		ret.FakeTits = sql.NullString{String: *performer.FakeTits, Valid: true}
+	}
+	if performer.CareerLength != nil {
+		ret.CareerLength = sql.NullString{String: *performer.CareerLength, Valid: true}
+	}
+	if performer.Tattoos != nil {
+		ret.Tattoos = sql.NullString{String: *performer.Tattoos, Valid: true}
+	}
+	if performer.Piercings != nil {
+		ret.Piercings = sql.NullString{String: *performer.Piercings, Valid: true}
+	}
+	if performer.Aliases != nil {
+		ret.Aliases = sql.NullString{String: *performer.Aliases, Valid: true}
+	}
+	if performer.Twitter != nil {
+		ret.Twitter = sql.NullString{String: *performer.Twitter, Valid: true}
+	}
+	if performer.Instagram != nil {
+		ret.Instagram = sql.NullString{String: *performer.Instagram, Valid: true}
+	}
+
+	return ret
+}

pkg/identify/performer_test.go

@@ -0,0 +1,329 @@
+package identify
+
+import (
+	"database/sql"
+	"errors"
+	"reflect"
+	"strconv"
+	"testing"
+
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/models/mocks"
+
+	"github.com/stretchr/testify/mock"
+)
+
+func Test_getPerformerID(t *testing.T) {
+	const (
+		emptyEndpoint = ""
+		endpoint      = "endpoint"
+	)
+	invalidStoredID := "invalidStoredID"
+	validStoredIDStr := "1"
+	validStoredID := 1
+	name := "name"
+
+	repo := mocks.NewTransactionManager()
+	repo.PerformerMock().On("Create", mock.Anything).Return(&models.Performer{
+		ID: validStoredID,
+	}, nil)
+
+	type args struct {
+		endpoint      string
+		p             *models.ScrapedPerformer
+		createMissing bool
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *int
+		wantErr bool
+	}{
+		{
+			"no performer",
+			args{
+				emptyEndpoint,
+				&models.ScrapedPerformer{},
+				false,
+			},
+			nil,
+			false,
+		},
+		{
+			"invalid stored id",
+			args{
+				emptyEndpoint,
+				&models.ScrapedPerformer{
+					StoredID: &invalidStoredID,
+				},
+				false,
+			},
+			nil,
+			true,
+		},
+		{
+			"valid stored id",
+			args{
+				emptyEndpoint,
+				&models.ScrapedPerformer{
+					StoredID: &validStoredIDStr,
+				},
+				false,
+			},
+			&validStoredID,
+			false,
+		},
+		{
+			"nil stored not creating",
+			args{
+				emptyEndpoint,
+				&models.ScrapedPerformer{
+					Name: &name,
+				},
+				false,
+			},
+			nil,
+			false,
+		},
+		{
+			"nil name creating",
+			args{
+				emptyEndpoint,
+				&models.ScrapedPerformer{},
+				true,
+			},
+			nil,
+			false,
+		},
+		{
+			"valid name creating",
+			args{
+				emptyEndpoint,
+				&models.ScrapedPerformer{
+					Name: &name,
+				},
+				true,
+			},
+			&validStoredID,
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := getPerformerID(tt.args.endpoint, repo, tt.args.p, tt.args.createMissing)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("getPerformerID() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getPerformerID() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_createMissingPerformer(t *testing.T) {
+	emptyEndpoint := ""
+	validEndpoint := "validEndpoint"
+	invalidEndpoint := "invalidEndpoint"
+	remoteSiteID := "remoteSiteID"
+	validName := "validName"
+	invalidName := "invalidName"
+	performerID := 1
+
+	repo := mocks.NewTransactionManager()
+	repo.PerformerMock().On("Create", mock.MatchedBy(func(p models.Performer) bool {
+		return p.Name.String == validName
+	})).Return(&models.Performer{
+		ID: performerID,
+	}, nil)
+	repo.PerformerMock().On("Create", mock.MatchedBy(func(p models.Performer) bool {
+		return p.Name.String == invalidName
+	})).Return(nil, errors.New("error creating performer"))
+
+	repo.PerformerMock().On("UpdateStashIDs", performerID, []models.StashID{
+		{
+			Endpoint: invalidEndpoint,
+			StashID:  remoteSiteID,
+		},
+	}).Return(errors.New("error updating stash ids"))
+	repo.PerformerMock().On("UpdateStashIDs", performerID, []models.StashID{
+		{
+			Endpoint: validEndpoint,
+			StashID:  remoteSiteID,
+		},
+	}).Return(nil)
+
+	type args struct {
+		endpoint string
+		p        *models.ScrapedPerformer
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *int
+		wantErr bool
+	}{
+		{
+			"simple",
+			args{
+				emptyEndpoint,
+				&models.ScrapedPerformer{
+					Name: &validName,
+				},
+			},
+			&performerID,
+			false,
+		},
+		{
+			"error creating",
+			args{
+				emptyEndpoint,
+				&models.ScrapedPerformer{
+					Name: &invalidName,
+				},
+			},
+			nil,
+			true,
+		},
+		{
+			"valid stash id",
+			args{
+				validEndpoint,
+				&models.ScrapedPerformer{
+					Name:         &validName,
+					RemoteSiteID: &remoteSiteID,
+				},
+			},
+			&performerID,
+			false,
+		},
+		{
+			"invalid stash id",
+			args{
+				invalidEndpoint,
+				&models.ScrapedPerformer{
+					Name:         &validName,
+					RemoteSiteID: &remoteSiteID,
+				},
+			},
+			nil,
+			true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := createMissingPerformer(tt.args.endpoint, repo, tt.args.p)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("createMissingPerformer() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("createMissingPerformer() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_scrapedToPerformerInput(t *testing.T) {
+	name := "name"
+	md5 := "b068931cc450442b63f5b3d276ea4297"
+
+	var stringValues []string
+	for i := 0; i < 16; i++ {
+		stringValues = append(stringValues, strconv.Itoa(i))
+	}
+
+	upTo := 0
+	nextVal := func() *string {
+		ret := stringValues[upTo]
+		upTo = (upTo + 1) % len(stringValues)
+		return &ret
+	}
+
+	tests := []struct {
+		name      string
+		performer *models.ScrapedPerformer
+		want      models.Performer
+	}{
+		{
+			"set all",
+			&models.ScrapedPerformer{
+				Name:         &name,
+				Birthdate:    nextVal(),
+				DeathDate:    nextVal(),
+				Gender:       nextVal(),
+				Ethnicity:    nextVal(),
+				Country:      nextVal(),
+				EyeColor:     nextVal(),
+				HairColor:    nextVal(),
+				Height:       nextVal(),
+				Measurements: nextVal(),
+				FakeTits:     nextVal(),
+				CareerLength: nextVal(),
+				Tattoos:      nextVal(),
+				Piercings:    nextVal(),
+				Aliases:      nextVal(),
+				Twitter:      nextVal(),
+				Instagram:    nextVal(),
+			},
+			models.Performer{
+				Name:     models.NullString(name),
+				Checksum: md5,
+				Favorite: sql.NullBool{
+					Bool:  false,
+					Valid: true,
+				},
+				Birthdate: models.SQLiteDate{
+					String: *nextVal(),
+					Valid:  true,
+				},
+				DeathDate: models.SQLiteDate{
+					String: *nextVal(),
+					Valid:  true,
+				},
+				Gender:       models.NullString(*nextVal()),
+				Ethnicity:    models.NullString(*nextVal()),
+				Country:      models.NullString(*nextVal()),
+				EyeColor:     models.NullString(*nextVal()),
+				HairColor:    models.NullString(*nextVal()),
+				Height:       models.NullString(*nextVal()),
+				Measurements: models.NullString(*nextVal()),
+				FakeTits:     models.NullString(*nextVal()),
+				CareerLength: models.NullString(*nextVal()),
+				Tattoos:      models.NullString(*nextVal()),
+				Piercings:    models.NullString(*nextVal()),
+				Aliases:      models.NullString(*nextVal()),
+				Twitter:      models.NullString(*nextVal()),
+				Instagram:    models.NullString(*nextVal()),
+			},
+		},
+		{
+			"set none",
+			&models.ScrapedPerformer{
+				Name: &name,
+			},
+			models.Performer{
+				Name:     models.NullString(name),
+				Checksum: md5,
+				Favorite: sql.NullBool{
+					Bool:  false,
+					Valid: true,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := scrapedToPerformerInput(tt.performer)
+
+			// clear created/updated dates
+			got.CreatedAt = models.SQLiteTimestamp{}
+			got.UpdatedAt = got.CreatedAt
+
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("scrapedToPerformerInput() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/identify/scene.go

@@ -0,0 +1,251 @@
+package identify
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/utils"
+)
+
+type sceneRelationships struct {
+	repo         models.Repository
+	scene        *models.Scene
+	result       *scrapeResult
+	fieldOptions map[string]*models.IdentifyFieldOptionsInput
+}
+
+func (g sceneRelationships) studio() (*int64, error) {
+	existingID := g.scene.StudioID
+	fieldStrategy := g.fieldOptions["studio"]
+	createMissing := fieldStrategy != nil && utils.IsTrue(fieldStrategy.CreateMissing)
+
+	scraped := g.result.result.Studio
+	endpoint := g.result.source.RemoteSite
+
+	if scraped == nil || !shouldSetSingleValueField(fieldStrategy, existingID.Valid) {
+		return nil, nil
+	}
+
+	if scraped.StoredID != nil {
+		// existing studio, just set it
+		studioID, err := strconv.ParseInt(*scraped.StoredID, 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("error converting studio ID %s: %w", *scraped.StoredID, err)
+		}
+
+		// only return value if different to current
+		if existingID.Int64 != studioID {
+			return &studioID, nil
+		}
+	} else if createMissing {
+		return createMissingStudio(endpoint, g.repo, scraped)
+	}
+
+	return nil, nil
+}
+
+func (g sceneRelationships) performers(ignoreMale bool) ([]int, error) {
+	fieldStrategy := g.fieldOptions["performers"]
+	scraped := g.result.result.Performers
+
+	// just check if ignored
+	if len(scraped) == 0 || !shouldSetSingleValueField(fieldStrategy, false) {
+		return nil, nil
+	}
+
+	createMissing := fieldStrategy != nil && utils.IsTrue(fieldStrategy.CreateMissing)
+	strategy := models.IdentifyFieldStrategyMerge
+	if fieldStrategy != nil {
+		strategy = fieldStrategy.Strategy
+	}
+
+	repo := g.repo
+	endpoint := g.result.source.RemoteSite
+
+	var performerIDs []int
+	originalPerformerIDs, err := repo.Scene().GetPerformerIDs(g.scene.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error getting scene performers: %w", err)
+	}
+
+	if strategy == models.IdentifyFieldStrategyMerge {
+		// add to existing
+		performerIDs = originalPerformerIDs
+	}
+
+	for _, p := range scraped {
+		if ignoreMale && p.Gender != nil && strings.EqualFold(*p.Gender, models.GenderEnumMale.String()) {
+			continue
+		}
+
+		performerID, err := getPerformerID(endpoint, repo, p, createMissing)
+		if err != nil {
+			return nil, err
+		}
+
+		if performerID != nil {
+			performerIDs = utils.IntAppendUnique(performerIDs, *performerID)
+		}
+	}
+
+	// don't return if nothing was added
+	if utils.SliceSame(originalPerformerIDs, performerIDs) {
+		return nil, nil
+	}
+
+	return performerIDs, nil
+}
+
+func (g sceneRelationships) tags() ([]int, error) {
+	fieldStrategy := g.fieldOptions["tags"]
+	scraped := g.result.result.Tags
+	target := g.scene
+	r := g.repo
+
+	// just check if ignored
+	if len(scraped) == 0 || !shouldSetSingleValueField(fieldStrategy, false) {
+		return nil, nil
+	}
+
+	createMissing := fieldStrategy != nil && utils.IsTrue(fieldStrategy.CreateMissing)
+	strategy := models.IdentifyFieldStrategyMerge
+	if fieldStrategy != nil {
+		strategy = fieldStrategy.Strategy
+	}
+
+	var tagIDs []int
+	originalTagIDs, err := r.Scene().GetTagIDs(target.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error getting scene tags: %w", err)
+	}
+
+	if strategy == models.IdentifyFieldStrategyMerge {
+		// add to existing
+		tagIDs = originalTagIDs
+	}
+
+	for _, t := range scraped {
+		if t.StoredID != nil {
+			// existing tag, just add it
+			tagID, err := strconv.ParseInt(*t.StoredID, 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("error converting tag ID %s: %w", *t.StoredID, err)
+			}
+
+			tagIDs = utils.IntAppendUnique(tagIDs, int(tagID))
+		} else if createMissing {
+			now := time.Now()
+			created, err := r.Tag().Create(models.Tag{
+				Name:      t.Name,
+				CreatedAt: models.SQLiteTimestamp{Timestamp: now},
+				UpdatedAt: models.SQLiteTimestamp{Timestamp: now},
+			})
+			if err != nil {
+				return nil, fmt.Errorf("error creating tag: %w", err)
+			}
+
+			tagIDs = append(tagIDs, created.ID)
+		}
+	}
+
+	// don't return if nothing was added
+	if utils.SliceSame(originalTagIDs, tagIDs) {
+		return nil, nil
+	}
+
+	return tagIDs, nil
+}
+
+func (g sceneRelationships) stashIDs() ([]models.StashID, error) {
+	remoteSiteID := g.result.result.RemoteSiteID
+	fieldStrategy := g.fieldOptions["stash_ids"]
+	target := g.scene
+	r := g.repo
+
+	endpoint := g.result.source.RemoteSite
+
+	// just check if ignored
+	if remoteSiteID == nil || endpoint == "" || !shouldSetSingleValueField(fieldStrategy, false) {
+		return nil, nil
+	}
+
+	strategy := models.IdentifyFieldStrategyMerge
+	if fieldStrategy != nil {
+		strategy = fieldStrategy.Strategy
+	}
+
+	var originalStashIDs []models.StashID
+	var stashIDs []models.StashID
+	stashIDPtrs, err := r.Scene().GetStashIDs(target.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error getting scene tag: %w", err)
+	}
+
+	// convert existing to non-pointer types
+	for _, stashID := range stashIDPtrs {
+		originalStashIDs = append(originalStashIDs, *stashID)
+	}
+
+	if strategy == models.IdentifyFieldStrategyMerge {
+		// add to existing
+		stashIDs = originalStashIDs
+	}
+
+	for i, stashID := range stashIDs {
+		if endpoint == stashID.Endpoint {
+			// if stashID is the same, then don't set
+			if stashID.StashID == *remoteSiteID {
+				return nil, nil
+			}
+
+			// replace the stash id and return
+			stashID.StashID = *remoteSiteID
+			stashIDs[i] = stashID
+			return stashIDs, nil
+		}
+	}
+
+	// not found, create new entry
+	stashIDs = append(stashIDs, models.StashID{
+		StashID:  *remoteSiteID,
+		Endpoint: endpoint,
+	})
+
+	if utils.SliceSame(originalStashIDs, stashIDs) {
+		return nil, nil
+	}
+
+	return stashIDs, nil
+}
+
+func (g sceneRelationships) cover(ctx context.Context) ([]byte, error) {
+	scraped := g.result.result.Image
+	r := g.repo
+
+	if scraped == nil {
+		return nil, nil
+	}
+
+	// always overwrite if present
+	existingCover, err := r.Scene().GetCover(g.scene.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error getting scene cover: %w", err)
+	}
+
+	data, err := utils.ProcessImageInput(ctx, *scraped)
+	if err != nil {
+		return nil, fmt.Errorf("error processing image input: %w", err)
+	}
+
+	// only return if different
+	if !bytes.Equal(existingCover, data) {
+		return data, nil
+	}
+
+	return nil, nil
+}

pkg/identify/scene_test.go

@@ -0,0 +1,782 @@
+package identify
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"strconv"
+	"testing"
+
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/models/mocks"
+	"github.com/stashapp/stash/pkg/utils"
+	"github.com/stretchr/testify/mock"
+)
+
+func Test_sceneRelationships_studio(t *testing.T) {
+	validStoredID := "1"
+	var validStoredIDInt int64 = 1
+	invalidStoredID := "invalidStoredID"
+	createMissing := true
+
+	defaultOptions := &models.IdentifyFieldOptionsInput{
+		Strategy: models.IdentifyFieldStrategyMerge,
+	}
+
+	repo := mocks.NewTransactionManager()
+	repo.StudioMock().On("Create", mock.Anything).Return(&models.Studio{
+		ID: int(validStoredIDInt),
+	}, nil)
+
+	tr := sceneRelationships{
+		repo:         repo,
+		fieldOptions: make(map[string]*models.IdentifyFieldOptionsInput),
+	}
+
+	tests := []struct {
+		name         string
+		scene        *models.Scene
+		fieldOptions *models.IdentifyFieldOptionsInput
+		result       *models.ScrapedStudio
+		want         *int64
+		wantErr      bool
+	}{
+		{
+			"nil studio",
+			&models.Scene{},
+			defaultOptions,
+			nil,
+			nil,
+			false,
+		},
+		{
+			"ignore",
+			&models.Scene{},
+			&models.IdentifyFieldOptionsInput{
+				Strategy: models.IdentifyFieldStrategyIgnore,
+			},
+			&models.ScrapedStudio{
+				StoredID: &validStoredID,
+			},
+			nil,
+			false,
+		},
+		{
+			"invalid stored id",
+			&models.Scene{},
+			defaultOptions,
+			&models.ScrapedStudio{
+				StoredID: &invalidStoredID,
+			},
+			nil,
+			true,
+		},
+		{
+			"same stored id",
+			&models.Scene{
+				StudioID: models.NullInt64(validStoredIDInt),
+			},
+			defaultOptions,
+			&models.ScrapedStudio{
+				StoredID: &validStoredID,
+			},
+			nil,
+			false,
+		},
+		{
+			"different stored id",
+			&models.Scene{},
+			defaultOptions,
+			&models.ScrapedStudio{
+				StoredID: &validStoredID,
+			},
+			&validStoredIDInt,
+			false,
+		},
+		{
+			"no create missing",
+			&models.Scene{},
+			defaultOptions,
+			&models.ScrapedStudio{},
+			nil,
+			false,
+		},
+		{
+			"create missing",
+			&models.Scene{},
+			&models.IdentifyFieldOptionsInput{
+				Strategy:      models.IdentifyFieldStrategyMerge,
+				CreateMissing: &createMissing,
+			},
+			&models.ScrapedStudio{},
+			&validStoredIDInt,
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tr.scene = tt.scene
+			tr.fieldOptions["studio"] = tt.fieldOptions
+			tr.result = &scrapeResult{
+				result: &models.ScrapedScene{
+					Studio: tt.result,
+				},
+			}
+
+			got, err := tr.studio()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("sceneRelationships.studio() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("sceneRelationships.studio() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_sceneRelationships_performers(t *testing.T) {
+	const (
+		sceneID = iota
+		sceneWithPerformerID
+		errSceneID
+		existingPerformerID
+		validStoredIDInt
+	)
+	validStoredID := strconv.Itoa(validStoredIDInt)
+	invalidStoredID := "invalidStoredID"
+	createMissing := true
+	existingPerformerStr := strconv.Itoa(existingPerformerID)
+	validName := "validName"
+	female := models.GenderEnumFemale.String()
+	male := models.GenderEnumMale.String()
+
+	defaultOptions := &models.IdentifyFieldOptionsInput{
+		Strategy: models.IdentifyFieldStrategyMerge,
+	}
+
+	repo := mocks.NewTransactionManager()
+	repo.SceneMock().On("GetPerformerIDs", sceneID).Return(nil, nil)
+	repo.SceneMock().On("GetPerformerIDs", sceneWithPerformerID).Return([]int{existingPerformerID}, nil)
+	repo.SceneMock().On("GetPerformerIDs", errSceneID).Return(nil, errors.New("error getting IDs"))
+
+	tr := sceneRelationships{
+		repo:         repo,
+		fieldOptions: make(map[string]*models.IdentifyFieldOptionsInput),
+	}
+
+	tests := []struct {
+		name         string
+		sceneID      int
+		fieldOptions *models.IdentifyFieldOptionsInput
+		scraped      []*models.ScrapedPerformer
+		ignoreMale   bool
+		want         []int
+		wantErr      bool
+	}{
+		{
+			"ignore",
+			sceneID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy: models.IdentifyFieldStrategyIgnore,
+			},
+			[]*models.ScrapedPerformer{
+				{
+					StoredID: &validStoredID,
+				},
+			},
+			false,
+			nil,
+			false,
+		},
+		{
+			"none",
+			sceneID,
+			defaultOptions,
+			[]*models.ScrapedPerformer{},
+			false,
+			nil,
+			false,
+		},
+		{
+			"error getting ids",
+			errSceneID,
+			defaultOptions,
+			[]*models.ScrapedPerformer{
+				{},
+			},
+			false,
+			nil,
+			true,
+		},
+		{
+			"merge existing",
+			sceneWithPerformerID,
+			defaultOptions,
+			[]*models.ScrapedPerformer{
+				{
+					Name:     &validName,
+					StoredID: &existingPerformerStr,
+				},
+			},
+			false,
+			nil,
+			false,
+		},
+		{
+			"merge add",
+			sceneWithPerformerID,
+			defaultOptions,
+			[]*models.ScrapedPerformer{
+				{
+					Name:     &validName,
+					StoredID: &validStoredID,
+				},
+			},
+			false,
+			[]int{existingPerformerID, validStoredIDInt},
+			false,
+		},
+		{
+			"ignore male",
+			sceneID,
+			defaultOptions,
+			[]*models.ScrapedPerformer{
+				{
+					Name:     &validName,
+					StoredID: &validStoredID,
+					Gender:   &male,
+				},
+			},
+			true,
+			nil,
+			false,
+		},
+		{
+			"overwrite",
+			sceneWithPerformerID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy: models.IdentifyFieldStrategyOverwrite,
+			},
+			[]*models.ScrapedPerformer{
+				{
+					Name:     &validName,
+					StoredID: &validStoredID,
+				},
+			},
+			false,
+			[]int{validStoredIDInt},
+			false,
+		},
+		{
+			"ignore male (not male)",
+			sceneWithPerformerID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy: models.IdentifyFieldStrategyOverwrite,
+			},
+			[]*models.ScrapedPerformer{
+				{
+					Name:     &validName,
+					StoredID: &validStoredID,
+					Gender:   &female,
+				},
+			},
+			true,
+			[]int{validStoredIDInt},
+			false,
+		},
+		{
+			"error getting tag ID",
+			sceneID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy:      models.IdentifyFieldStrategyOverwrite,
+				CreateMissing: &createMissing,
+			},
+			[]*models.ScrapedPerformer{
+				{
+					Name:     &validName,
+					StoredID: &invalidStoredID,
+				},
+			},
+			false,
+			nil,
+			true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tr.scene = &models.Scene{
+				ID: tt.sceneID,
+			}
+			tr.fieldOptions["performers"] = tt.fieldOptions
+			tr.result = &scrapeResult{
+				result: &models.ScrapedScene{
+					Performers: tt.scraped,
+				},
+			}
+
+			got, err := tr.performers(tt.ignoreMale)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("sceneRelationships.performers() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("sceneRelationships.performers() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_sceneRelationships_tags(t *testing.T) {
+	const (
+		sceneID = iota
+		sceneWithTagID
+		errSceneID
+		existingID
+		validStoredIDInt
+	)
+	validStoredID := strconv.Itoa(validStoredIDInt)
+	invalidStoredID := "invalidStoredID"
+	createMissing := true
+	existingIDStr := strconv.Itoa(existingID)
+	validName := "validName"
+	invalidName := "invalidName"
+
+	defaultOptions := &models.IdentifyFieldOptionsInput{
+		Strategy: models.IdentifyFieldStrategyMerge,
+	}
+
+	repo := mocks.NewTransactionManager()
+	repo.SceneMock().On("GetTagIDs", sceneID).Return(nil, nil)
+	repo.SceneMock().On("GetTagIDs", sceneWithTagID).Return([]int{existingID}, nil)
+	repo.SceneMock().On("GetTagIDs", errSceneID).Return(nil, errors.New("error getting IDs"))
+
+	repo.TagMock().On("Create", mock.MatchedBy(func(p models.Tag) bool {
+		return p.Name == validName
+	})).Return(&models.Tag{
+		ID: validStoredIDInt,
+	}, nil)
+	repo.TagMock().On("Create", mock.MatchedBy(func(p models.Tag) bool {
+		return p.Name == invalidName
+	})).Return(nil, errors.New("error creating tag"))
+
+	tr := sceneRelationships{
+		repo:         repo,
+		fieldOptions: make(map[string]*models.IdentifyFieldOptionsInput),
+	}
+
+	tests := []struct {
+		name         string
+		sceneID      int
+		fieldOptions *models.IdentifyFieldOptionsInput
+		scraped      []*models.ScrapedTag
+		want         []int
+		wantErr      bool
+	}{
+		{
+			"ignore",
+			sceneID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy: models.IdentifyFieldStrategyIgnore,
+			},
+			[]*models.ScrapedTag{
+				{
+					StoredID: &validStoredID,
+				},
+			},
+			nil,
+			false,
+		},
+		{
+			"none",
+			sceneID,
+			defaultOptions,
+			[]*models.ScrapedTag{},
+			nil,
+			false,
+		},
+		{
+			"error getting ids",
+			errSceneID,
+			defaultOptions,
+			[]*models.ScrapedTag{
+				{},
+			},
+			nil,
+			true,
+		},
+		{
+			"merge existing",
+			sceneWithTagID,
+			defaultOptions,
+			[]*models.ScrapedTag{
+				{
+					Name:     validName,
+					StoredID: &existingIDStr,
+				},
+			},
+			nil,
+			false,
+		},
+		{
+			"merge add",
+			sceneWithTagID,
+			defaultOptions,
+			[]*models.ScrapedTag{
+				{
+					Name:     validName,
+					StoredID: &validStoredID,
+				},
+			},
+			[]int{existingID, validStoredIDInt},
+			false,
+		},
+		{
+			"overwrite",
+			sceneWithTagID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy: models.IdentifyFieldStrategyOverwrite,
+			},
+			[]*models.ScrapedTag{
+				{
+					Name:     validName,
+					StoredID: &validStoredID,
+				},
+			},
+			[]int{validStoredIDInt},
+			false,
+		},
+		{
+			"error getting tag ID",
+			sceneID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy: models.IdentifyFieldStrategyOverwrite,
+			},
+			[]*models.ScrapedTag{
+				{
+					Name:     validName,
+					StoredID: &invalidStoredID,
+				},
+			},
+			nil,
+			true,
+		},
+		{
+			"create missing",
+			sceneID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy:      models.IdentifyFieldStrategyOverwrite,
+				CreateMissing: &createMissing,
+			},
+			[]*models.ScrapedTag{
+				{
+					Name: validName,
+				},
+			},
+			[]int{validStoredIDInt},
+			false,
+		},
+		{
+			"error creating",
+			sceneID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy:      models.IdentifyFieldStrategyOverwrite,
+				CreateMissing: &createMissing,
+			},
+			[]*models.ScrapedTag{
+				{
+					Name: invalidName,
+				},
+			},
+			nil,
+			true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tr.scene = &models.Scene{
+				ID: tt.sceneID,
+			}
+			tr.fieldOptions["tags"] = tt.fieldOptions
+			tr.result = &scrapeResult{
+				result: &models.ScrapedScene{
+					Tags: tt.scraped,
+				},
+			}
+
+			got, err := tr.tags()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("sceneRelationships.tags() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("sceneRelationships.tags() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_sceneRelationships_stashIDs(t *testing.T) {
+	const (
+		sceneID = iota
+		sceneWithStashID
+		errSceneID
+		existingID
+		validStoredIDInt
+	)
+	existingEndpoint := "existingEndpoint"
+	newEndpoint := "newEndpoint"
+	remoteSiteID := "remoteSiteID"
+	newRemoteSiteID := "newRemoteSiteID"
+
+	defaultOptions := &models.IdentifyFieldOptionsInput{
+		Strategy: models.IdentifyFieldStrategyMerge,
+	}
+
+	repo := mocks.NewTransactionManager()
+	repo.SceneMock().On("GetStashIDs", sceneID).Return(nil, nil)
+	repo.SceneMock().On("GetStashIDs", sceneWithStashID).Return([]*models.StashID{
+		{
+			StashID:  remoteSiteID,
+			Endpoint: existingEndpoint,
+		},
+	}, nil)
+	repo.SceneMock().On("GetStashIDs", errSceneID).Return(nil, errors.New("error getting IDs"))
+
+	tr := sceneRelationships{
+		repo:         repo,
+		fieldOptions: make(map[string]*models.IdentifyFieldOptionsInput),
+	}
+
+	tests := []struct {
+		name         string
+		sceneID      int
+		fieldOptions *models.IdentifyFieldOptionsInput
+		endpoint     string
+		remoteSiteID *string
+		want         []models.StashID
+		wantErr      bool
+	}{
+		{
+			"ignore",
+			sceneID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy: models.IdentifyFieldStrategyIgnore,
+			},
+			newEndpoint,
+			&remoteSiteID,
+			nil,
+			false,
+		},
+		{
+			"no endpoint",
+			sceneID,
+			defaultOptions,
+			"",
+			&remoteSiteID,
+			nil,
+			false,
+		},
+		{
+			"no site id",
+			sceneID,
+			defaultOptions,
+			newEndpoint,
+			nil,
+			nil,
+			false,
+		},
+		{
+			"error getting ids",
+			errSceneID,
+			defaultOptions,
+			newEndpoint,
+			&remoteSiteID,
+			nil,
+			true,
+		},
+		{
+			"merge existing",
+			sceneWithStashID,
+			defaultOptions,
+			existingEndpoint,
+			&remoteSiteID,
+			nil,
+			false,
+		},
+		{
+			"merge existing new value",
+			sceneWithStashID,
+			defaultOptions,
+			existingEndpoint,
+			&newRemoteSiteID,
+			[]models.StashID{
+				{
+					StashID:  newRemoteSiteID,
+					Endpoint: existingEndpoint,
+				},
+			},
+			false,
+		},
+		{
+			"merge add",
+			sceneWithStashID,
+			defaultOptions,
+			newEndpoint,
+			&newRemoteSiteID,
+			[]models.StashID{
+				{
+					StashID:  remoteSiteID,
+					Endpoint: existingEndpoint,
+				},
+				{
+					StashID:  newRemoteSiteID,
+					Endpoint: newEndpoint,
+				},
+			},
+			false,
+		},
+		{
+			"overwrite",
+			sceneWithStashID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy: models.IdentifyFieldStrategyOverwrite,
+			},
+			newEndpoint,
+			&newRemoteSiteID,
+			[]models.StashID{
+				{
+					StashID:  newRemoteSiteID,
+					Endpoint: newEndpoint,
+				},
+			},
+			false,
+		},
+		{
+			"overwrite same",
+			sceneWithStashID,
+			&models.IdentifyFieldOptionsInput{
+				Strategy: models.IdentifyFieldStrategyOverwrite,
+			},
+			existingEndpoint,
+			&remoteSiteID,
+			nil,
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tr.scene = &models.Scene{
+				ID: tt.sceneID,
+			}
+			tr.fieldOptions["stash_ids"] = tt.fieldOptions
+			tr.result = &scrapeResult{
+				source: ScraperSource{
+					RemoteSite: tt.endpoint,
+				},
+				result: &models.ScrapedScene{
+					RemoteSiteID: tt.remoteSiteID,
+				},
+			}
+
+			got, err := tr.stashIDs()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("sceneRelationships.stashIDs() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("sceneRelationships.stashIDs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_sceneRelationships_cover(t *testing.T) {
+	const (
+		sceneID = iota
+		sceneWithStashID
+		errSceneID
+		existingID
+		validStoredIDInt
+	)
+	existingData := []byte("existingData")
+	newData := []byte("newData")
+	const base64Prefix = "data:image/png;base64,"
+	existingDataEncoded := base64Prefix + utils.GetBase64StringFromData(existingData)
+	newDataEncoded := base64Prefix + utils.GetBase64StringFromData(newData)
+	invalidData := newDataEncoded + "!!!"
+
+	repo := mocks.NewTransactionManager()
+	repo.SceneMock().On("GetCover", sceneID).Return(existingData, nil)
+	repo.SceneMock().On("GetCover", errSceneID).Return(nil, errors.New("error getting cover"))
+
+	tr := sceneRelationships{
+		repo:         repo,
+		fieldOptions: make(map[string]*models.IdentifyFieldOptionsInput),
+	}
+
+	tests := []struct {
+		name    string
+		sceneID int
+		image   *string
+		want    []byte
+		wantErr bool
+	}{
+		{
+			"nil image",
+			sceneID,
+			nil,
+			nil,
+			false,
+		},
+		{
+			"different image",
+			sceneID,
+			&newDataEncoded,
+			newData,
+			false,
+		},
+		{
+			"same image",
+			sceneID,
+			&existingDataEncoded,
+			nil,
+			false,
+		},
+		{
+			"error getting scene cover",
+			errSceneID,
+			&newDataEncoded,
+			nil,
+			true,
+		},
+		{
+			"invalid data",
+			sceneID,
+			&invalidData,
+			nil,
+			true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tr.scene = &models.Scene{
+				ID: tt.sceneID,
+			}
+			tr.result = &scrapeResult{
+				result: &models.ScrapedScene{
+					Image: tt.image,
+				},
+			}
+
+			got, err := tr.cover(context.TODO())
+			if (err != nil) != tt.wantErr {
+				t.Errorf("sceneRelationships.cover() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("sceneRelationships.cover() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}