diff --git a/.dockerignore b/.dockerignore index 990c8acd4..954624f3c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,16 +15,10 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out -# Packr2 artifacts -**/*-packr.go - # GraphQL generated output pkg/models/generated_*.go ui/v2.5/src/core/generated-*.tsx -# packr generated files -*-packr.go - #### # Jetbrains #### @@ -60,4 +54,4 @@ ui/v2.5/build stash dist -docker \ No newline at end of file +docker diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e963bd17e..8c2cf3452 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,8 +8,12 @@ on: release: types: [ published ] +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + env: - COMPILER_IMAGE: stashapp/compiler:4 + COMPILER_IMAGE: stashapp/compiler:5 jobs: build: @@ -52,34 +56,40 @@ jobs: run: | mkdir -p .go-cache docker run -d --name build --mount type=bind,source="$(pwd)",target=/stash,consistency=delegated --mount type=bind,source="$(pwd)/.go-cache",target=/root/.cache/go-build,consistency=delegated -w /stash $COMPILER_IMAGE tail -f /dev/null - + - name: Pre-install run: docker exec -t build /bin/bash -c "make pre-ui" - name: Generate run: docker exec -t build /bin/bash -c "make generate" - + - name: Validate UI # skip UI validation for pull requests if UI is unchanged if: ${{ github.event_name != 'pull_request' || steps.cache-ui.outputs.cache-hit != 'true' }} - run: docker exec -t build /bin/bash -c "make ui-validate" + run: docker exec -t build /bin/bash -c "make validate-frontend" - # TODO: Replace with `make validate` once `revive` is bundled in COMPILER_IMAGE - - name: Validate - run: docker exec -t build /bin/bash -c "make fmt-check vet it" + # Static validation happens in the linter workflow in parallel to this workflow + # Run Dynamic validation here, to make sure we pass all the projects integration tests + # + # create UI file so that the embed doesn't fail + - name: Test Backend + run: | + mkdir -p ui/v2.5/build + touch ui/v2.5/build/index.html + docker exec -t build /bin/bash -c "make it" - name: Build UI # skip UI build for pull requests if UI is unchanged (UI was cached) # this means that the build version/time may be incorrect if the UI is # not changed in a pull request if: ${{ github.event_name != 'pull_request' || steps.cache-ui.outputs.cache-hit != 'true' }} - run: docker exec -t build /bin/bash -c "make ui-only" + run: docker exec -t build /bin/bash -c "make ui" - name: Compile for all supported platforms run: | - docker exec -t build /bin/bash -c "make packr" docker exec -t build /bin/bash -c "make cross-compile-windows" - docker exec -t build /bin/bash -c "make cross-compile-osx" + docker exec -t build /bin/bash -c "make cross-compile-osx-intel" + docker exec -t build /bin/bash -c "make cross-compile-osx-applesilicon" docker exec -t build /bin/bash -c "make cross-compile-linux" docker exec -t build /bin/bash -c "make cross-compile-linux-arm64v8" docker exec -t build /bin/bash -c "make cross-compile-linux-arm32v7" @@ -94,7 +104,7 @@ jobs: sha1sum dist/stash-* | sed 's/dist\///g' | tee -a CHECKSUMS_SHA1 echo "STASH_VERSION=$(git describe --tags --exclude latest_develop)" >> $GITHUB_ENV echo "RELEASE_DATE=$(date +'%Y-%m-%d %H:%M:%S %Z')" >> $GITHUB_ENV - + - name: Upload Windows binary # only upload binaries for pull requests if: ${{ github.event_name == 'pull_request' && github.base_ref != 'refs/heads/develop' && github.base_ref != 'refs/heads/master'}} @@ -118,13 +128,13 @@ jobs: with: name: stash-linux path: dist/stash-linux - + - name: Update latest_develop tag if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }} run : git tag -f latest_develop; git push -f --tags - name: Development Release - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }} + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }} uses: marvinpinto/action-automatic-releases@v1.1.2 with: repo_token: "${{ secrets.GITHUB_TOKEN }}" @@ -133,13 +143,14 @@ jobs: title: "${{ env.STASH_VERSION }}: Latest development build" files: | dist/stash-osx + dist/stash-osx-applesilicon dist/stash-win.exe dist/stash-linux dist/stash-linux-arm64v8 dist/stash-linux-arm32v7 dist/stash-pi CHECKSUMS_SHA1 - + - name: Master release if: ${{ github.event_name == 'release' && github.ref != 'refs/tags/latest_develop' }} uses: meeDamian/github-release@2.0 @@ -148,6 +159,7 @@ jobs: allow_override: true files: | dist/stash-osx + dist/stash-osx-applesilicon dist/stash-win.exe dist/stash-linux dist/stash-linux-arm64v8 @@ -155,7 +167,7 @@ jobs: dist/stash-pi CHECKSUMS_SHA1 gzip: false - + - name: Development Docker if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }} env: @@ -163,7 +175,7 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | - docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64 + docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64 docker info docker buildx create --name builder --use docker buildx inspect --bootstrap @@ -177,7 +189,7 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} run: | - docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64 + docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64 docker info docker buildx create --name builder --use docker buildx inspect --bootstrap diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 000000000..ab5c50010 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,60 @@ +name: Lint (golangci-lint) +on: + push: + tags: + - v* + branches: + - master + - develop + pull_request: + +env: + COMPILER_IMAGE: stashapp/compiler:5 + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Checkout + run: git fetch --prune --unshallow --tags + + - name: Pull compiler image + run: docker pull $COMPILER_IMAGE + + - name: Start build container + run: | + mkdir -p .go-cache + docker run -d --name build --mount type=bind,source="$(pwd)",target=/stash,consistency=delegated --mount type=bind,source="$(pwd)/.go-cache",target=/root/.cache/go-build,consistency=delegated -w /stash $COMPILER_IMAGE tail -f /dev/null + + - name: Generate Backend + run: docker exec -t build /bin/bash -c "make generate-backend" + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.42.1 + + # Optional: working directory, useful for monorepos + # working-directory: somedir + + # Optional: golangci-lint command line arguments. + args: --modules-download-mode=vendor --timeout=3m + + # Optional: show only new issues if it's a pull request. The default value is `false`. + # only-new-issues: true + + # Optional: if set to true then the action will use pre-installed Go. + # skip-go-installation: true + + # Optional: if set to true then the action don't cache or restore ~/go/pkg. + # skip-pkg-cache: true + + # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. + # skip-build-cache: true + + - name: Cleanup build container + run: docker rm -f -v build diff --git a/.gitignore b/.gitignore index d6c2eea2c..8ea22a10b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,16 +15,10 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out -# Packr2 artifacts -**/*-packr.go - # GraphQL generated output pkg/models/generated_*.go ui/v2.5/src/core/generated-*.tsx -# packr generated files -*-packr.go - #### # Jetbrains #### diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..ca296c1dd --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,82 @@ +# options for analysis running +run: + timeout: 3m + modules-download-mode: vendor + +linters: + disable-all: true + enable: + # Default set of linters from golangci-lint + - deadcode + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + # Linters added by the stash project + # - bodyclose + - dogsled + # - errorlint + # - exhaustive + - exportloopref + # - goconst + # - gocritic + # - goerr113 + - gofmt + # - gosec + # - ifshort + - misspell + # - nakedret + # - noctx + # - paralleltest + - revive + - rowserrcheck + - sqlclosecheck + +linters-settings: + gofmt: + simplify: false + + revive: + ignore-generated-header: true + severity: error + confidence: 0.8 + error-code: 1 + warning-code: 1 + rules: + - name: blank-imports + disabled: true + - name: context-as-argument + - name: context-keys-type + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: exported + disabled: true + - name: if-return + disabled: true + - name: increment-decrement + - name: var-naming + disabled: true + - name: var-declaration + - name: package-comments + - name: range + - name: receiver-naming + - name: time-naming + - name: unexported-return + disabled: true + - name: indent-error-flow + disabled: true + - name: errorf + - name: empty-block + disabled: true + - name: superfluous-else + - name: unused-parameter + disabled: true + - name: unreachable-code + - name: redefines-builtin-id \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index 238fea065..7c10ffeb8 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -29,6 +29,18 @@ builds: - darwin goarch: - amd64 + - binary: stash-osx-applesilicon + env: + - CGO_ENABLED=1 + - CC=oa64-clang + - CXX=oa64-clang++ + flags: + - -tags + - extended + goos: + - darwin + goarch: + - arm64 - binary: stash-linux env: - CGO_ENABLED=1 diff --git a/.travis.yml.disabled b/.travis.yml.disabled index 3641d0dd9..91a12196f 100644 --- a/.travis.yml.disabled +++ b/.travis.yml.disabled @@ -5,12 +5,9 @@ git: depth: false language: go go: -- 1.13.x +- 1.17.x services: - docker -env: - global: - - GO111MODULE=on before_install: - set -e # Configure environment so changes are picked up when the Docker daemon is restarted after upgrading @@ -41,7 +38,7 @@ script: #- make lint - make fmt-check vet it after_success: -- docker pull stashapp/compiler:4 +- docker pull stashapp/compiler:5 - sh ./scripts/cross-compile.sh - git describe --tags --exclude latest_develop | tee CHECKSUMS_SHA1 - sha1sum dist/stash-* | sed 's/dist\///g' | tee -a CHECKSUMS_SHA1 @@ -62,6 +59,7 @@ deploy: secure: tGJ2q62CfPdayid2qEtW2aGRhMgCl3lBXYYQqp3eH0vFgIIf6cs7IDX7YC/x3XKMEQ/iMLZmtCXZvSTqNrD6Sk7MSnt30GIs+4uxIZDnnd8mV5X3K4n4gjD+NAORc4DrQBvUGrYMKJsR5gtkH0nu6diWb1o1If7OiJEuCPRhrmQYcza7NUdABnA9Z2wn2RNUV9Ga33WUCqLMEU5GtNBlfQPiP/khCQrqn/ocR6wUjYut3J6YagzqH4wsfJi3glHyWtowcNIw1LZi5zFxHD/bRBT4Tln7yypkjWNq9eQILA6i6kRUGf7ggyTx26/k8n4tnu+QD0vVh4EcjlThpU/LGyUXzKrrxjRwaDZnM0oYxg5AfHcBuAiAdo0eWnV3lEWRfTJMIVb9MPf4qDmzR4RREfB5OXOxwq3ODeCcJE8sTIMD/wBPZrlqS/QrRpND2gn2X4snkVukN9t9F4CMTFMtVSzFV7TDJW5E5Lq6VEExulteQhs6kcK9NRPNAaLgRQAw7X9kVWfDtiGUP+fE2i8F9Bo8bm7sOT5O5VPMPykx3EgeNg1IqIgMTCsMlhMJT4xBJoQUgmd2wWyf3Ryw+P+sFgdb5Sd7+lFgJBjMUUoOxMxAOiEgdFvCXcr+/Udyz2RdtetU1/6VzXzLPcKOw0wubZeBkISqu7o9gpfdMP9Eq00= file: - dist/stash-osx + - dist/stash-osx-applesilicon - dist/stash-win.exe - dist/stash-linux - dist/stash-linux-arm64v8 @@ -89,6 +87,7 @@ deploy: secure: tGJ2q62CfPdayid2qEtW2aGRhMgCl3lBXYYQqp3eH0vFgIIf6cs7IDX7YC/x3XKMEQ/iMLZmtCXZvSTqNrD6Sk7MSnt30GIs+4uxIZDnnd8mV5X3K4n4gjD+NAORc4DrQBvUGrYMKJsR5gtkH0nu6diWb1o1If7OiJEuCPRhrmQYcza7NUdABnA9Z2wn2RNUV9Ga33WUCqLMEU5GtNBlfQPiP/khCQrqn/ocR6wUjYut3J6YagzqH4wsfJi3glHyWtowcNIw1LZi5zFxHD/bRBT4Tln7yypkjWNq9eQILA6i6kRUGf7ggyTx26/k8n4tnu+QD0vVh4EcjlThpU/LGyUXzKrrxjRwaDZnM0oYxg5AfHcBuAiAdo0eWnV3lEWRfTJMIVb9MPf4qDmzR4RREfB5OXOxwq3ODeCcJE8sTIMD/wBPZrlqS/QrRpND2gn2X4snkVukN9t9F4CMTFMtVSzFV7TDJW5E5Lq6VEExulteQhs6kcK9NRPNAaLgRQAw7X9kVWfDtiGUP+fE2i8F9Bo8bm7sOT5O5VPMPykx3EgeNg1IqIgMTCsMlhMJT4xBJoQUgmd2wWyf3Ryw+P+sFgdb5Sd7+lFgJBjMUUoOxMxAOiEgdFvCXcr+/Udyz2RdtetU1/6VzXzLPcKOw0wubZeBkISqu7o9gpfdMP9Eq00= file: - dist/stash-osx + - dist/stash-osx-applesilicon - dist/stash-win.exe - dist/stash-linux - dist/stash-linux-arm64v8 diff --git a/Makefile b/Makefile index fae4673b7..0fe2eedbc 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ endif ifdef IS_WIN SEPARATOR := && SET := set -else +else SEPARATOR := ; SET := export endif @@ -23,9 +23,8 @@ ifdef OUTPUT endif export CGO_ENABLED = 1 -export GO111MODULE = on -.PHONY: release pre-build install clean +.PHONY: release pre-build release: generate ui build-release @@ -44,14 +43,15 @@ endif build: pre-build $(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/pkg/api.version=$(STASH_VERSION)' -X 'github.com/stashapp/stash/pkg/api.buildstamp=$(BUILD_DATE)' -X 'github.com/stashapp/stash/pkg/api.githash=$(GITHASH)') - go build $(OUTPUT) -mod=vendor -v -tags "sqlite_omit_load_extension osusergo netgo" -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS)" + go build $(OUTPUT) -mod=vendor -v -tags "sqlite_omit_load_extension osusergo netgo" $(GO_BUILD_FLAGS) -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS)" # strips debug symbols from the release build -# consider -trimpath in go build if we move to go 1.13+ build-release: EXTRA_LDFLAGS := -s -w +build-release: GO_BUILD_FLAGS := -trimpath build-release: build build-release-static: EXTRA_LDFLAGS := -extldflags=-static -s -w +build-release-static: GO_BUILD_FLAGS := -trimpath build-release-static: build # cross-compile- targets should be run within the compiler docker container @@ -62,13 +62,21 @@ cross-compile-windows: export CXX := x86_64-w64-mingw32-g++ cross-compile-windows: OUTPUT := -o dist/stash-win.exe cross-compile-windows: build-release-static -cross-compile-osx: export GOOS := darwin -cross-compile-osx: export GOARCH := amd64 -cross-compile-osx: export CC := o64-clang -cross-compile-osx: export CXX := o64-clang++ -cross-compile-osx: OUTPUT := -o dist/stash-osx +cross-compile-osx-intel: export GOOS := darwin +cross-compile-osx-intel: export GOARCH := amd64 +cross-compile-osx-intel: export CC := o64-clang +cross-compile-osx-intel: export CXX := o64-clang++ +cross-compile-osx-intel: OUTPUT := -o dist/stash-osx # can't use static build for OSX -cross-compile-osx: build-release +cross-compile-osx-intel: build-release + +cross-compile-osx-applesilicon: export GOOS := darwin +cross-compile-osx-applesilicon: export GOARCH := arm64 +cross-compile-osx-applesilicon: export CC := oa64e-clang +cross-compile-osx-applesilicon: export CXX := oa64e-clang++ +cross-compile-osx-applesilicon: OUTPUT := -o dist/stash-osx-applesilicon +# can't use static build for OSX +cross-compile-osx-applesilicon: build-release cross-compile-linux: export GOOS := linux cross-compile-linux: export GOARCH := amd64 @@ -95,20 +103,26 @@ cross-compile-pi: export CC := arm-linux-gnueabi-gcc cross-compile-pi: OUTPUT := -o dist/stash-pi cross-compile-pi: build-release-static -cross-compile-all: cross-compile-windows cross-compile-osx cross-compile-linux cross-compile-linux-arm64v8 cross-compile-linux-arm32v7 cross-compile-pi - -install: - packr2 install - -clean: - packr2 clean +cross-compile-all: + make cross-compile-windows + make cross-compile-osx-intel + make cross-compile-osx-applesilicon + make cross-compile-linux + make cross-compile-linux-arm64v8 + make cross-compile-linux-arm32v7 + make cross-compile-pi # Regenerates GraphQL files -.PHONY: generate -generate: - go generate -mod=vendor +generate: generate-backend generate-frontend + +.PHONY: generate-frontend +generate-frontend: cd ui/v2.5 && yarn run gqlgen +.PHONY: generate-backend +generate-backend: + go generate -mod=vendor + # Regenerates stash-box client files .PHONY: generate-stash-box-client generate-stash-box-client: @@ -119,23 +133,13 @@ generate-stash-box-client: fmt: go fmt ./... -# Ensures that changed files have had gofmt run on them -.PHONY: fmt-check -fmt-check: - sh ./scripts/check-gofmt.sh - -# Runs go vet on the project's source code. -.PHONY: vet -vet: - go vet -mod=vendor ./... - .PHONY: lint lint: - revive -config revive.toml -exclude ./vendor/... ./... + golangci-lint run # runs unit tests - excluding integration tests .PHONY: test -test: +test: go test -mod=vendor ./... # runs all tests - including integration tests @@ -148,23 +152,19 @@ it: generate-test-mocks: go run -mod=vendor github.com/vektra/mockery/v2 --dir ./pkg/models --name '.*ReaderWriter' --outpkg mocks --output ./pkg/models/mocks -# installs UI dependencies. Run when first cloning repository, or if UI +# installs UI dependencies. Run when first cloning repository, or if UI # dependencies have changed .PHONY: pre-ui pre-ui: cd ui/v2.5 && yarn install --frozen-lockfile -.PHONY: ui-only -ui-only: pre-build +.PHONY: ui +ui: pre-build $(SET) REACT_APP_DATE="$(BUILD_DATE)" $(SEPARATOR) \ $(SET) REACT_APP_GITHASH=$(GITHASH) $(SEPARATOR) \ $(SET) REACT_APP_STASH_VERSION=$(STASH_VERSION) $(SEPARATOR) \ cd ui/v2.5 && yarn build -.PHONY: ui -ui: ui-only - packr2 - .PHONY: ui-start ui-start: pre-build $(SET) REACT_APP_DATE="$(BUILD_DATE)" $(SEPARATOR) \ @@ -181,12 +181,19 @@ fmt-ui: ui-validate: cd ui/v2.5 && yarn run validate -# just repacks the packr files - use when updating migrations and packed files without -# rebuilding the UI -.PHONY: packr -packr: - packr2 - # runs all of the tests and checks required for a PR to be accepted .PHONY: validate -validate: ui-validate fmt-check vet lint it +validate: validate-frontend validate-backend + +# runs all of the frontend PR-acceptance steps +.PHONY: validate-frontend +validate-frontend: ui-validate + +# runs all of the backend PR-acceptance steps +.PHONY: validate-backend +validate-backend: lint it + +# locally builds and tags a 'stash/build' docker image +.PHONY: docker-build +docker-build: + docker build -t stash/build -f docker/build/x86_64/Dockerfile . diff --git a/README.md b/README.md index 74ab1301c..16ee10641 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # Stash -[![Build Status](https://travis-ci.org/stashapp/stash.svg?branch=master)](https://travis-ci.org/stashapp/stash) [![Go Report Card](https://goreportcard.com/badge/github.com/stashapp/stash)](https://goreportcard.com/report/github.com/stashapp/stash) [![Discord](https://img.shields.io/discord/559159668438728723.svg?logo=discord)](https://discord.gg/2TsNFKt) @@ -68,6 +67,10 @@ This command would need customizing for your environment. [This link](https://s Once you have a certificate and key file name them `stash.crt` and `stash.key` and place them in the same directory as the `config.yml` file, or the `~/.stash` directory. Stash detects these and starts up using HTTPS rather than HTTP. +## Basepath rewriting + +The basepath defaults to `/`. When running stash via a reverse proxy in a subpath, the basepath can be changed by having the reverse proxy pass `X-Forwarded-Prefix` (and optionally `X-Forwarded-Port`) headers. When detects these headers, it alters the basepath URL of the UI. + # Customization ## Themes and CSS Customization @@ -90,13 +93,10 @@ For issues not addressed there, there are a few options. ## Pre-requisites * [Go](https://golang.org/dl/) -* [Revive](https://github.com/mgechev/revive) - Configurable linter - * Go Install: `go get github.com/mgechev/revive` -* [Packr2](https://github.com/gobuffalo/packr/) - Static asset bundler - * Go Install: `go get github.com/gobuffalo/packr/v2/packr2` - * [Binary Download](https://github.com/gobuffalo/packr/releases) +* [GolangCI](https://golangci-lint.run/) - A meta-linter which runs several linters in parallel + * To install, follow the [local installation instructions](https://golangci-lint.run/usage/install/#local-installation) * [Yarn](https://yarnpkg.com/en/docs/install) - Yarn package manager - * Run `yarn install --frozen-lockfile` in the `stash/ui/v2.5` folder (before running make generate for first time). + * Run `yarn install --frozen-lockfile` in the `stash/ui/v2.5` folder (before running make generate for first time). NOTE: You may need to run the `go get` commands outside the project directory to avoid modifying the projects module file. @@ -112,8 +112,7 @@ TODO 2. Download and install [MingW](https://sourceforge.net/projects/mingw-w64/) 3. Search for "advanced system settings" and open the system properties dialog. 1. Click the `Environment Variables` button - 2. Add `GO111MODULE=on` - 3. Under system variables find the `Path`. Edit and add `C:\Program Files\mingw-w64\*\mingw64\bin` (replace * with the correct path). + 2. Under system variables find the `Path`. Edit and add `C:\Program Files\mingw-w64\*\mingw64\bin` (replace * with the correct path). NOTE: The `make` command in Windows will be `mingw32-make` with MingW. @@ -121,21 +120,19 @@ NOTE: The `make` command in Windows will be `mingw32-make` with MingW. * `make generate` - Generate Go and UI GraphQL files * `make build` - Builds the binary (make sure to build the UI as well... see below) +* `make docker-build` - Locally builds and tags a complete 'stash/build' docker image * `make pre-ui` - Installs the UI dependencies. Only needs to be run once before building the UI for the first time, or if the dependencies are updated -* `make fmt-ui` - Formats the UI source code. -* `make ui` - Builds the frontend and the packr2 files -* `make packr` - Generate packr2 files (sub-target of `ui`. Use to regenerate packr2 files without rebuilding UI) -* `make vet` - Run `go vet` -* `make lint` - Run the linter +* `make fmt-ui` - Formats the UI source code +* `make ui` - Builds the frontend +* `make lint` - Run the linter on the backend * `make fmt` - Run `go fmt` -* `make fmt-check` - Ensure changed files are formatted correctly * `make it` - Run the unit and integration tests * `make validate` - Run all of the tests and checks required to submit a PR * `make ui-start` - Runs the UI in development mode. Requires a running stash server to connect to. Stash port can be changed from the default of `9999` with environment variable `REACT_APP_PLATFORM_PORT`. ## Building a release -1. Run `make generate` to create generated files +1. Run `make generate` to create generated files 2. Run `make ui` to compile the frontend 3. Run `make build` to build the executable for your current platform @@ -151,7 +148,7 @@ command to open a bash shell to the container to poke around: Stash can be profiled using the `--cpuprofile ` command line flag. -The resulting file can then be used with pprof as follows: +The resulting file can then be used with pprof as follows: `go tool pprof ` diff --git a/docker/build/x86_64/Dockerfile b/docker/build/x86_64/Dockerfile index 3a2d8a198..2ffcf7050 100644 --- a/docker/build/x86_64/Dockerfile +++ b/docker/build/x86_64/Dockerfile @@ -1,61 +1,43 @@ -# this dockerfile must be built from the top-level stash directory -# ie from top=level stash: +# This dockerfile must be built from the top-level stash directory +# ie from top-level stash: # docker build -t stash/build -f docker/build/x86_64/Dockerfile . -FROM golang:1.13.15 as compiler - -RUN apt-get update && apt-get install -y apt-transport-https -RUN curl -sL https://deb.nodesource.com/setup_lts.x | bash - - -# prevent caching of the key -ADD https://dl.yarnpkg.com/debian/pubkey.gpg yarn.gpg -RUN cat yarn.gpg | apt-key add - && \ - echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ - rm yarn.gpg - -RUN apt-get update && \ - apt-get install -y nodejs yarn xz-utils --no-install-recommends || exit 1; \ - rm -rf /var/lib/apt/lists/*; - -ENV PACKR2_VERSION=2.0.2 -ENV PACKR2_SHA=f95ff4c96d7a28813220df030ad91700b8464fe292ab3e1dc9582305c2a338d2 -ENV PACKR2_DOWNLOAD_FILE=packr_${PACKR2_VERSION}_linux_amd64.tar.gz -ENV PACKR2_DOWNLOAD_URL=https://github.com/gobuffalo/packr/releases/download/v${PACKR2_VERSION}/${PACKR2_DOWNLOAD_FILE} +# Build Frontend +FROM node:alpine as frontend +RUN apk add --no-cache make git +## cache node_modules separately +COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/ +WORKDIR /stash +RUN yarn --cwd ui/v2.5 install --frozen-lockfile. +COPY Makefile /stash/ +COPY ./.git /stash/.git +COPY ./graphql /stash/graphql/ +COPY ./ui /stash/ui/ +RUN make generate-frontend +RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui +# Build Backend +FROM golang:1.17-alpine as backend +RUN apk add --no-cache xz make alpine-sdk +## install ffmpeg WORKDIR / -RUN wget ${PACKR2_DOWNLOAD_URL}; \ - echo "$PACKR2_SHA $PACKR2_DOWNLOAD_FILE" | sha256sum -c - || exit 1; \ - tar -xzf $PACKR2_DOWNLOAD_FILE -C /usr/bin/ packr2; \ - rm $PACKR2_DOWNLOAD_FILE; - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - RUN wget -O /ffmpeg.tar.xz https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \ tar xf /ffmpeg.tar.xz && \ rm ffmpeg.tar.xz && \ mv /ffmpeg*/ /ffmpeg/ - -# copy the ui yarn stuff so that it doesn't get rebuilt every time -COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/ - WORKDIR /stash -RUN yarn --cwd ui/v2.5 install --frozen-lockfile - -COPY . /stash/ -ENV GO111MODULE=on - -RUN make generate -RUN make ui +COPY ./go* ./*.go Makefile gqlgen.yml .gqlgenc.yml /stash/ +COPY ./scripts /stash/scripts/ +COPY ./vendor /stash/vendor/ +COPY ./pkg /stash/pkg/ +COPY --from=frontend /stash /stash/ +RUN make generate-backend RUN make build -FROM ubuntu:20.04 as app - -RUN apt-get update && apt-get -y install ca-certificates -COPY --from=compiler /stash/stash /ffmpeg/ffmpeg /ffmpeg/ffprobe /usr/bin/ - +# Final Runnable Image +FROM alpine:latest +RUN apk add --no-cache ca-certificates vips-tools +COPY --from=backend /stash/stash /ffmpeg/ffmpeg /ffmpeg/ffprobe /usr/bin/ ENV STASH_CONFIG_FILE=/root/.stash/config.yml - EXPOSE 9999 -CMD ["stash"] - - +ENTRYPOINT ["stash"] \ No newline at end of file diff --git a/docker/ci/x86_64/Dockerfile b/docker/ci/x86_64/Dockerfile index 9eb5d57d4..7e2fd24fa 100644 --- a/docker/ci/x86_64/Dockerfile +++ b/docker/ci/x86_64/Dockerfile @@ -11,7 +11,7 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then BIN=stash-pi; \ ENV DEBIAN_FRONTEND=noninteractive RUN apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-pip && pip3 install cloudscraper FROM ubuntu:20.04 as app -run apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-mechanicalsoup ffmpeg && rm -rf /var/lib/apt/lists/* +run apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-mechanicalsoup ffmpeg libvips-tools && rm -rf /var/lib/apt/lists/* COPY --from=prep /stash /usr/bin/ COPY --from=prep /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages diff --git a/docker/compiler/Dockerfile b/docker/compiler/Dockerfile index 5f3f4b4c6..96fc9f161 100644 --- a/docker/compiler/Dockerfile +++ b/docker/compiler/Dockerfile @@ -1,11 +1,6 @@ -FROM golang:1.13.15 +FROM golang:1.17 -LABEL maintainer="stashappdev@gmail.com" - -ENV PACKR2_VERSION=2.0.2 -ENV PACKR2_SHA=f95ff4c96d7a28813220df030ad91700b8464fe292ab3e1dc9582305c2a338d2 -ENV PACKR2_DOWNLOAD_FILE=packr_${PACKR2_VERSION}_linux_amd64.tar.gz -ENV PACKR2_DOWNLOAD_URL=https://github.com/gobuffalo/packr/releases/download/v${PACKR2_VERSION}/${PACKR2_DOWNLOAD_FILE} +LABEL maintainer="https://discord.gg/2TsNFKt" # Install tools RUN apt-get update && apt-get install -y apt-transport-https @@ -18,10 +13,10 @@ RUN cat yarn.gpg | apt-key add - && \ rm yarn.gpg RUN apt-get update && \ - apt-get install -y automake autogen \ + apt-get install -y automake autogen cmake \ libtool libxml2-dev uuid-dev libssl-dev bash \ - patch make tar xz-utils bzip2 gzip sed cpio \ - gcc-8-multilib gcc-mingw-w64 g++-mingw-w64 clang llvm-dev \ + patch make tar xz-utils bzip2 gzip zlib1g-dev sed cpio \ + gcc-10-multilib gcc-mingw-w64 g++-mingw-w64 clang llvm-dev \ gcc-arm-linux-gnueabi libc-dev-armel-cross linux-libc-dev-armel-cross \ gcc-arm-linux-gnueabihf libc-dev-armhf-cross \ gcc-aarch64-linux-gnu libc-dev-arm64-cross \ @@ -29,21 +24,22 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/*; # Cross compile setup -ENV OSX_SDK_VERSION 10.11 +ENV OSX_SDK_VERSION 11.3 ENV OSX_SDK_DOWNLOAD_FILE=MacOSX${OSX_SDK_VERSION}.sdk.tar.xz -ENV OSX_SDK_DOWNLOAD_URL=https://github.com/ndeloof/golang-cross/raw/113fix/${OSX_SDK_DOWNLOAD_FILE} -ENV OSX_SDK_SHA=98cdd56e0f6c1f9e1af25e11dd93d2e7d306a4aa50430a2bc6bc083ac67efbb8 +ENV OSX_SDK_DOWNLOAD_URL=https://github.com/phracker/MacOSX-SDKs/releases/download/${OSX_SDK_VERSION}/${OSX_SDK_DOWNLOAD_FILE} +ENV OSX_SDK_SHA=cd4f08a75577145b8f05245a2975f7c81401d75e9535dcffbb879ee1deefcbf4 ENV OSX_SDK MacOSX$OSX_SDK_VERSION.sdk ENV OSX_NDK_X86 /usr/local/osx-ndk-x86 RUN wget ${OSX_SDK_DOWNLOAD_URL} RUN echo "$OSX_SDK_SHA $OSX_SDK_DOWNLOAD_FILE" | sha256sum -c - || exit 1; \ - git clone https://github.com/tpoechtrager/osxcross.git && \ - git -C osxcross checkout a9317c18a3a457ca0a657f08cc4d0d43c6cf8953 || exit 1; \ - mv $OSX_SDK_DOWNLOAD_FILE osxcross/tarballs/ && \ - UNATTENDED=yes SDK_VERSION=${OSX_SDK_VERSION} OSX_VERSION_MIN=10.9 osxcross/build.sh || exit 1; \ - mv osxcross/target $OSX_NDK_X86; \ - rm -rf osxcross; + git clone https://github.com/tpoechtrager/osxcross.git; \ + mv $OSX_SDK_DOWNLOAD_FILE osxcross/tarballs/ + +RUN UNATTENDED=yes SDK_VERSION=${OSX_SDK_VERSION} OSX_VERSION_MIN=10.10 osxcross/build.sh || exit 1; +RUN cp osxcross/target/lib/* /usr/lib/ ; \ + mv osxcross/target $OSX_NDK_X86; \ + rm -rf osxcross; ENV PATH $OSX_NDK_X86/bin:$PATH @@ -51,14 +47,6 @@ RUN mkdir -p /root/.ssh; \ chmod 0700 /root/.ssh; \ ssh-keyscan github.com > /root/.ssh/known_hosts; -RUN wget ${PACKR2_DOWNLOAD_URL}; \ - echo "$PACKR2_SHA $PACKR2_DOWNLOAD_FILE" | sha256sum -c - || exit 1; \ - tar -xzf $PACKR2_DOWNLOAD_FILE -C /usr/bin/ packr2; \ - rm $PACKR2_DOWNLOAD_FILE; - -CMD ["packr2", "version"] - - # Notes for self: # Windows: # GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ go build -ldflags "-extldflags '-static'" -tags extended @@ -66,4 +54,4 @@ CMD ["packr2", "version"] # Darwin # CC=o64-clang CXX=o64-clang++ GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build -tags extended -# env GO111MODULE=on goreleaser --config=goreleaser-extended.yml --skip-publish --skip-validate --rm-dist --release-notes=temp/0.48-relnotes-ready.md +# env goreleaser --config=goreleaser-extended.yml --skip-publish --skip-validate --rm-dist --release-notes=temp/0.48-relnotes-ready.md diff --git a/docker/compiler/Makefile b/docker/compiler/Makefile index 6f444e73d..5c4ed71ba 100644 --- a/docker/compiler/Makefile +++ b/docker/compiler/Makefile @@ -1,6 +1,6 @@ user=stashapp repo=compiler -version=4 +version=5 latest: docker build -t ${user}/${repo}:latest . diff --git a/docker/compiler/README.md b/docker/compiler/README.md index abe4060b5..d25b4a6cd 100644 --- a/docker/compiler/README.md +++ b/docker/compiler/README.md @@ -1,3 +1,5 @@ Modified from https://github.com/bep/dockerfiles/tree/master/ci-goreleaser -When the dockerfile is changed, the version number should be incremented in the Makefile and the new version tag should be pushed to docker hub. The `scripts/cross-compile.sh` script should also be updated to use the new version number tag, and `.travis.yml` needs to be updated to pull the correct image tag. \ No newline at end of file +When the dockerfile is changed, the version number should be incremented in the Makefile and the new version tag should be pushed to docker hub. The `scripts/cross-compile.sh` script should also be updated to use the new version number tag, and `.travis.yml` needs to be updated to pull the correct image tag. + +A MacOS univeral binary can be created using `lipo -create -output stash-osx-universal stash-osx stash-osx-applesilicon`, available in the image. \ No newline at end of file diff --git a/docker/compiler/create_osx_sdk.sh b/docker/compiler/create_osx_sdk.sh deleted file mode 100644 index 7920f6e0e..000000000 --- a/docker/compiler/create_osx_sdk.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -TMP=$(mktemp -d /tmp/XXXXXXXXXXX) -SDK="MacOSX10.11.sdk" - -mkdir -p $TMP/$SDK/usr/include/c++ - -cp -rf /Applications/Xcode7.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/$SDK $TMP &>/dev/null || true -cp -rf /Applications/Xcode7.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1 $TMP/$SDK/usr/include/c++ || exit -1 - -tar -C $TMP -czf $SDK.tar.gz $SDK diff --git a/docker/develop/x86_64/Dockerfile b/docker/develop/x86_64/Dockerfile deleted file mode 100644 index c2efef3a1..000000000 --- a/docker/develop/x86_64/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM ubuntu:20.04 as prep -LABEL MAINTAINER="https://discord.gg/Uz29ny" - -RUN apt-get update && \ - apt-get -y install curl xz-utils && \ - apt-get autoclean -y && \ - rm -rf /var/lib/apt/lists/* -WORKDIR / -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -# added " to end of stash-linux clause so that it doesn't pick up the arm builds -RUN curl -L -o /stash $(curl -s https://api.github.com/repos/stashapp/stash/releases/tags/latest_develop | awk '/browser_download_url/ && /stash-linux"/' | sed -e 's/.*: "\(.*\)"/\1/') && \ - chmod +x /stash - -RUN curl --http1.1 -o /ffmpeg.tar.xz https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \ - tar xf /ffmpeg.tar.xz && \ - rm ffmpeg.tar.xz && \ - mv /ffmpeg*/ /ffmpeg/ - -FROM ubuntu:20.04 as app -RUN apt-get update && apt-get -y install ca-certificates -COPY --from=prep /stash /ffmpeg/ffmpeg /ffmpeg/ffprobe /usr/bin/ - -ENV STASH_CONFIG_FILE=/root/.stash/config.yml - -EXPOSE 9999 -CMD ["stash"] diff --git a/docker/production/README.md b/docker/production/README.md index 87d47142d..40fe157bc 100644 --- a/docker/production/README.md +++ b/docker/production/README.md @@ -1,53 +1,37 @@ -# Docker install on Ubuntu 18.04 -Installing StashApp can likely work on others if your OS either has it's own package manager or comes shipped with Docker and docker-compose. +# Docker Installation (for most 64-bit GNU/Linux systems) +StashApp is supported on most systems that support Docker and docker-compose. Your OS likely ships with or makes available the necessary packages. ## Dependencies -The goal is to avoid as many dependencies as possible so for now the only pre-requisites you are required to have are `curl`, `docker`, and `docker-compose` for the most part your understanding of the technologies can be superficial so long as you can follow commands and are open to reading a bit you should be fine. +Only `docker` and `docker-compose` are required. For the most part your understanding of the technologies can be superficial. So long as you can follow commands and are open to reading a bit, you should be fine. + +Installation instructions are available below, and if your distrobution's repository ships a current version of docker, you may use that. +https://docs.docker.com/engine/install/ ### Docker +Docker is effectively a cross-platform software package repository. It allows you to ship an entire environment in what's referred to as a container. Containers are intended to hold everything that is needed to run an application from one place to another, making it easy for everyone along the way to reproduce the environment. -Docker is effectively the cross-platform software package repository it allows you to ship an entire environment in what's referred to as a container. Containers are intended to hold everything that is needed to ship what's required to run an application from one place to another with a degree of a standard that makes it easy for everyone along the way to reproduce the environment for their step in the chain. +The StashApp docker container ships with everything you need to automatically build and run stash, including ffmpeg. -The other side of docker is it brings everything that we would typically have to teach you about the individual components of your soon to be installed StashApp and ffmpeg, docker-compose wraps it up nicely in a handful of easy to follow steps that should result in the same environment on everyone's host. +### docker-compose +Docker Compose lets you specify how and where to run your containers, and to manage their environment. The docker-compose.yml file in this folder gets you a fully working instance of StashApp exactly as you would need it to have a reasonable instance for testing / developing on. If you are deploying a live instance for production, a reverse proxy (such as NGINX or Traefik) is recommended, but not required. -The installation method we recommend is via the `docker.com` website however if your specific operating system's repository versions are at the latest along with docker you should be good to launch with you using whatever instructions you wish. The version of Docker we used in our deployment for testing this process was `Docker version 17.05.0-ce, build 89658be` however any versions later than this will be sufficient. At the writing of this tutorial, this was not the latest version of Docker. - -#### Just the link to installation instructions, please -Instructions for installing on Ubuntu are at the link that follows: -https://docs.docker.com/install/linux/docker-ce/ubuntu/ - -If you plan on using other versions of OS you should at least aim to be a Linux base with an x86_64 CPU and the appropriate minimum version of the dependencies. - -### Docker-compose -Docker Compose's role in this deployment is to get you a fully working instance of StashApp exactly as you would need it to have a reasonable instance for testing / developing on, you could technically deploy a live instance with this, but without a reverse proxy, is not recommended. You are encouraged to learn how to use the Docker-Compose format, but it's not a required prerequisite for getting this running you need to have it installed successfully. - -Install Docker Compose via this guide below, and it is essential if you're using an older version of Linux to use the official documentation from Docker.com because you require the more recent version of docker-compose at least version 3.4 aka 1.22.0 or newer. - -#### Just the link to installation instructions, please -https://docs.docker.com/compose/install/ - -### Install curl -This one's easy, copy paste. - -``` -apt update -y && \ -apt install -f curl -``` +The latest version is always recommended. ### Get the docker-compose.yml file -Now you can either navigate to the [docker-compose.yml](https://raw.githubusercontent.com/stashapp/stash/master/docker/production/docker-compose.yml) in the repository, OR you can make your Linux console do it for you with this. +Now you can either navigate to the [docker-compose.yml](https://raw.githubusercontent.com/stashapp/stash/master/docker/production/docker-compose.yml) in the repository, or if you have curl, you can make your Linux console do it for you: ``` -curl -o ~/docker-compose.yml https://raw.githubusercontent.com/stashapp/stash/master/docker/production/docker-compose.yml +mkdir stashapp && cd stashapp +curl -o docker-compose.yml https://raw.githubusercontent.com/stashapp/stash/master/docker/production/docker-compose.yml ``` -Once you have that file where you want it, you can either modify the settings as you please OR you can run the following to get it up and running instantly. +Once you have that file where you want it, modify the settings as you please, and then run: ``` -cd ~ && docker-compose up -d +docker-compose up -d ``` -Installing StashApp this way will by default bind stash to port 9999 or in web browser terms. http://YOURIP:9999 or if you're doing this on your machine locally which is the only recommended production version of this container as is with no security configurations set at all is http://localhost:9999 +Installing StashApp this way will by default bind stash to port 9999. This is available in your web browser locally at http://localhost:9999 or on your network as http://YOUR-LOCAL-IP:9999 Good luck and have fun! diff --git a/docker/production/docker-compose.yml b/docker/production/docker-compose.yml index 1d20eb42c..837e3c95c 100644 --- a/docker/production/docker-compose.yml +++ b/docker/production/docker-compose.yml @@ -4,9 +4,13 @@ version: '3.4' services: stash: image: stashapp/stash:latest + container_name: stash restart: unless-stopped + ## the container's port must be the same with the STASH_PORT in the environment section ports: - "9999:9999" + ## If you intend to use stash's DLNA functionality uncomment the below network mode and comment out the above ports section + # network_mode: host logging: driver: "json-file" options: @@ -17,12 +21,14 @@ services: - STASH_GENERATED=/generated/ - STASH_METADATA=/metadata/ - STASH_CACHE=/cache/ + ## Adjust below to change default port (9999) + - STASH_PORT=9999 volumes: - /etc/localtime:/etc/localtime:ro ## Adjust below paths (the left part) to your liking. ## E.g. you can change ./config:/root/.stash to ./stash:/root/.stash - ## Keep configs here. + ## Keep configs, scrapers, and plugins here. - ./config:/root/.stash ## Point this at your collection. - ./data:/data diff --git a/docker/production/x86_64/Dockerfile b/docker/production/x86_64/Dockerfile index 95a2516ed..607cd1fbb 100644 --- a/docker/production/x86_64/Dockerfile +++ b/docker/production/x86_64/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:20.04 as prep -LABEL MAINTAINER="leopere [at] nixc [dot] us" +LABEL MAINTAINER="https://discord.gg/2TsNFKt" RUN apt-get update && \ apt-get -y install curl xz-utils && \ diff --git a/go.mod b/go.mod index 08de5b6be..d2cd351a6 100644 --- a/go.mod +++ b/go.mod @@ -8,21 +8,18 @@ require ( github.com/chromedp/cdproto v0.0.0-20210622022015-fe1827b46b84 github.com/chromedp/chromedp v0.7.3 github.com/corona10/goimagehash v1.0.3 - github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/disintegration/imaging v1.6.0 github.com/fvbommel/sortorder v1.0.2 github.com/go-chi/chi v4.0.2+incompatible - github.com/gobuffalo/logger v1.0.4 // indirect - github.com/gobuffalo/packr/v2 v2.8.1 - github.com/golang-migrate/migrate/v4 v4.3.1 + github.com/golang-jwt/jwt/v4 v4.0.0 + github.com/golang-migrate/migrate/v4 v4.15.0-beta.1 github.com/gorilla/securecookie v1.1.1 github.com/gorilla/sessions v1.2.0 github.com/gorilla/websocket v1.4.2 github.com/h2non/filetype v1.0.8 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a - github.com/jmoiron/sqlx v1.2.0 + github.com/jmoiron/sqlx v1.3.1 github.com/json-iterator/go v1.1.9 - github.com/karrick/godirwalk v1.16.1 // indirect github.com/mattn/go-sqlite3 v1.14.6 github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007 github.com/remeh/sizedwaitgroup v1.0.0 @@ -33,21 +30,68 @@ require ( github.com/spf13/afero v1.2.0 // indirect github.com/spf13/pflag v1.0.3 github.com/spf13/viper v1.7.0 - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.6.1 github.com/tidwall/gjson v1.8.1 github.com/tidwall/pretty v1.2.0 // indirect github.com/vektah/gqlparser/v2 v2.0.1 github.com/vektra/mockery/v2 v2.2.1 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 + golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect - golang.org/x/tools v0.0.0-20200915031644-64986481280e // indirect + golang.org/x/tools v0.1.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect - gopkg.in/yaml.v2 v2.3.0 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/agnivade/levenshtein v1.1.0 // indirect + github.com/antchfx/xpath v1.1.6 // indirect + github.com/chromedp/sysutil v1.0.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.4.7 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.1.0-rc.5 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.0 // indirect + github.com/hashicorp/golang-lru v0.5.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/magiconair/properties v1.8.1 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matryer/moq v0.0.0-20200106131100-75d0ddfc0007 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect + github.com/pelletier/go-toml v1.7.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/zerolog v1.18.0 // indirect + github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/spf13/cast v1.3.0 // indirect + github.com/spf13/cobra v1.0.0 // indirect + github.com/spf13/jwalterweatherman v1.0.0 // indirect + github.com/stretchr/objx v0.2.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/tidwall/match v1.0.3 // indirect + github.com/urfave/cli/v2 v2.1.1 // indirect + github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e // indirect + go.uber.org/atomic v1.6.0 // indirect + golang.org/x/mod v0.4.1 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gopkg.in/ini.v1 v1.51.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) replace git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999 -go 1.13 +go 1.17 diff --git a/go.sum b/go.sum index 8e5faa378..457db0ce0 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,62 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/spanner v1.18.0/go.mod h1:LvAjUXPeJRGNuGpikMULjhLj/t9cRvdc+fxRoLiugXA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/gqlgen v0.12.2 h1:aOdpsiCycFtCnAv8CAI1exnKrIDHMqtMzQoXeTziY4o= github.com/99designs/gqlgen v0.12.2/go.mod h1:7zdGo6ry9u1YBp/qlb2uxSU5Mt2jQKLcBETQiKk+Bxo= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= +github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Yamashou/gqlgenc v0.0.0-20200902035953-4dbef3551953 h1:+iPJDL28FxZhEdtJ9qykrMt/oDiOvlzTa0zV06nUcFM= github.com/Yamashou/gqlgenc v0.0.0-20200902035953-4dbef3551953/go.mod h1:kaTsk10p2hJWwrB2t7vMsk1lXj9KAHaDYRtJQiB+Ick= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= @@ -45,7 +78,8 @@ github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/arrow/go/arrow v0.0.0-20200601151325-b2287a20f230/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= +github.com/apache/arrow/go/arrow v0.0.0-20210521153258-78c88a9f517b/go.mod h1:R4hW3Ug0s+n4CUsWHKOj00Pu01ZqU4x/hSF5kXUcXKQ= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -53,14 +87,42 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v1.3.2/go.mod h1:7OaACgj2SX3XGWnrIjGlJM22h6yD6MEWKvm7levnnM8= +github.com/aws/aws-sdk-go-v2 v1.6.0/go.mod h1:tI4KhsR5VkzlUa2DZAdwx7wCAYGwkZZ1H31PYrBFx1w= +github.com/aws/aws-sdk-go-v2/config v1.1.5/go.mod h1:P3F1hku7qzC81txjwXnwOM6Ex6ezkU6+/557Teyb64E= +github.com/aws/aws-sdk-go-v2/config v1.3.0/go.mod h1:lOxzHWDt/k7MMidA/K8DgXL4+ynnZYsDq65Qhs/l3dg= +github.com/aws/aws-sdk-go-v2/credentials v1.1.5/go.mod h1:Ir1R6tPiR1/2y1hes8yOijFMz54hzSmgcmCDo6F45Qc= +github.com/aws/aws-sdk-go-v2/credentials v1.2.1/go.mod h1:Rfvim1eZTC9W5s8YJyYYtl1KMk6e8fHv+wMRQGO4Ru0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.6/go.mod h1:0+fWMitrmIpENiY8/1DyhdYPUCAPvd9UNz9mtCsEoLQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.1.1/go.mod h1:GTXAhrxHQOj9N+J5tYVjwt+rpRyy/42qLjlgw9pz1a0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.1.2/go.mod h1:Azf567f5wBUfUbwpyJJnLM/geFFIzEulGR30L+nQZOE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.2.1/go.mod h1:2JOqaBP3I6TEm27NLb11UiD9j4HZsJ+EW4N7vCf8WGQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.0.0/go.mod h1:g3XMXuxvqSMUjnsXXp/960152w0wFS4CXVYgQaSVOHE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.4/go.mod h1:BCfU3Uo2fhKcMZFp9zU5QQGQxqWCOYmZ/27Dju3S/do= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.1.0/go.mod h1:zdjOOy0ojUn3iNELo6ycIHSMCp4xUbycSHfb8PnbbyM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.6/go.mod h1:L0KWr0ASo83PRZu9NaZaDsw3koS6PspKv137DMDZjHo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.1.1/go.mod h1:2+ehJPkdIdl46VCj67Emz/EH2hpebHZtaLdzqg+sWOI= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.2.2/go.mod h1:nnutjMLuna0s3GVY/MAkpLX03thyNER06gXvnMAPj5g= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.3.1/go.mod h1:IpjxfORBAFfkMM0VEx5gPPnEy6WV4Hk0F/+zb/SUWyw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.5.0/go.mod h1:uwA7gs93Qcss43astPUb1eq4RyceNmYWAQjZFDOAMLo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.8.0/go.mod h1:zHCjYoODbYRLz/iFicYswq1gRoxBnHvpY5h2Vg3/tJ4= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.5/go.mod h1:bpGz0tidC4y39sZkQSkpO/J0tzWCMXHbw6FZ0j1GkWM= +github.com/aws/aws-sdk-go-v2/service/sso v1.2.1/go.mod h1:VimPFPltQ/920i1X0Sb0VJBROLIHkDg2MNP10D46OGs= +github.com/aws/aws-sdk-go-v2/service/sts v1.2.2/go.mod h1:ssRzzJ2RZOVuKj2Vx1YE7ypfil/BIlgmQnCSW4DistU= +github.com/aws/aws-sdk-go-v2/service/sts v1.4.1/go.mod h1:G9osDWA52WQ38BDcj65VY1cNmcAQXAXTsE8IWH8j81w= +github.com/aws/smithy-go v1.3.1/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.4.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= +github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chromedp/cdproto v0.0.0-20210526005521-9e51b9051fd0/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U= github.com/chromedp/cdproto v0.0.0-20210622022015-fe1827b46b84 h1:Xxl4imt7LA3SbkrlIH5mm+mbzsv0tpHomLASTPINlvQ= @@ -69,58 +131,65 @@ github.com/chromedp/chromedp v0.7.3 h1:FvgJICfjvXtDX+miuMUY0NHuY8zQvjS/TcEQEG6Ld github.com/chromedp/chromedp v0.7.3/go.mod h1:9gC521Yzgrk078Ulv6KIgG7hJ2x9aWrxMBBobTFk30A= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= +github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/corona10/goimagehash v1.0.3 h1:NZM518aKLmoNluluhfHGxT3LGOnrojrxhGn63DR/CZA= github.com/corona10/goimagehash v1.0.3/go.mod h1:VkvE0mLn84L4aF8vCb6mafVajEb6QYMHl2ZJLn0mOGI= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= -github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg= -github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= -github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00/go.mod h1:olo7eAdKwJdXxb55TKGLiJ6xt1H0/tiiRCWKVLmtjY4= -github.com/cznic/lldb v1.1.0/go.mod h1:FIZVUmYUVhPwRiPzL8nD/mpFcJ/G7SSXjjXYG4uRI3A= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/ql v1.2.0/go.mod h1:FbpzhyZrqr0PVlK6ury+PoW3T0ODUV22OeWIxcaOrSE= -github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= -github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= -github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/trifles v0.0.0-20190318185328-a8d75aae118c/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c1yc= +github.com/dhui/dktest v0.3.4/go.mod h1:4m4n6lmXlmVfESth7mzdcv8nBI5mOb5UROPqjM02csU= github.com/disintegration/imaging v1.6.0 h1:nVPXRUUQ36Z7MNf0O77UzgnOb1mkMMor7lmJMJXc/mA= github.com/disintegration/imaging v1.6.0/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7lkkUojDSR247MQ= -github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v0.7.3-0.20190108045446-77df18c24acf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v17.12.0-ce-rc1.0.20210128214336-420b1d36250f+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsouza/fake-gcs-server v1.7.0/go.mod h1:5XIRs4YvwNbNoz+1JF8j6KLAyDh7RHGAyAK3EP2EsNk= +github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo= github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -130,20 +199,40 @@ github.com/go-chi/chi v3.3.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxm github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/logger v1.0.3/go.mod h1:SoeejUwldiS7ZsyCBphOGURmWdwUFXs0J7TCjEhjKxM= -github.com/gobuffalo/logger v1.0.4 h1:HFJRqL7AmL4QNvQb9Grss9sDz+3u02VBgAoR03A7q4o= -github.com/gobuffalo/logger v1.0.4/go.mod h1:/GRUdWb+gM3shxj0P5jiV6ecVS3X0aboJvl+hBu0HeE= -github.com/gobuffalo/packd v1.0.0 h1:6ERZvJHfe24rfFmA9OaoKBdC7+c9sydrytMg8SdFGBM= -github.com/gobuffalo/packd v1.0.0/go.mod h1:6VTc4htmJRFB7u1m/4LeMTWjFoYrUiBkU9Fdec9hrhI= -github.com/gobuffalo/packr/v2 v2.8.1 h1:tkQpju6i3EtMXJ9uoF5GT6kB+LMTimDWD8Xvbz6zDVA= -github.com/gobuffalo/packr/v2 v2.8.1/go.mod h1:c/PLlOuTU+p3SybaJATW3H6lX/iK7xEz5OeMf+NnJpg= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= @@ -151,39 +240,91 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.1.0-rc.5 h1:QOAag7FoBaBYYHRqzqkhhd8fq5RTubvI4v3Ft/gDVVQ= github.com/gobwas/ws v1.1.0-rc.5/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang-migrate/migrate/v4 v4.3.1 h1:3eR1NY+pplX+m6yJ1fQf5dFWX3fBgUtZfDiaS/kJVu4= -github.com/golang-migrate/migrate/v4 v4.3.1/go.mod h1:mJ89KBgbXmM3P49BqOxRL3riNF/ATlg5kMhm17GA0dE= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-migrate/migrate/v4 v4.15.0-beta.1 h1:3iUwrd6V9oIzNc6TQdp4SLYNjQV1DXOK/E7cjaq7zbo= +github.com/golang-migrate/migrate/v4 v4.15.0-beta.1/go.mod h1:QOmbm9b62AcsxBz7VbwJf+3mqgAyVrdKx7AQ8T9m5og= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github/v35 v35.2.0/go.mod h1:s0515YVTI+IMrDoy9Y4pHt9ShGpzHvHO8rZ7L7acgvs= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -191,9 +332,11 @@ github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ= @@ -214,8 +357,9 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -233,15 +377,64 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= -github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.6.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o= github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jmoiron/sqlx v1.3.1 h1:aLN7YINNZ7cYOPK3QC83dbM6KT0NMqVMw961TqrejlE= +github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -249,48 +442,64 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= -github.com/karrick/godirwalk v1.15.8/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= -github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kshvakov/clickhouse v1.3.5/go.mod h1:DMzX7FxRymoNkVgizH0DWAL8Cur7wHLgx3MUnGwJqpE= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= +github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= -github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= -github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= -github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= -github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/moq v0.0.0-20200106131100-75d0ddfc0007 h1:reVOUXwnhsYv/8UqjvhrMOu5CNT9UapHFLbQ2JcXsmg= github.com/matryer/moq v0.0.0-20200106131100-75d0ddfc0007/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -304,6 +513,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -312,28 +522,38 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mutecomm/go-sqlcipher/v4 v4.4.0/go.mod h1:PyN04SaWalavxRGH9E8ZftG6Ju7rsPrGmQRjrEaVpiY= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA= github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007 h1:Ohgj9L0EYOgXxkDp+bczlMBiulwmqYzQpvQNUdtt3oc= github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007/go.mod h1:wKCOWMb6iNlvKiOToY2cNuaovSXvIiv1zDi9QDR7aGQ= +github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.4/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.7/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -342,35 +562,34 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac h1:kYPjbEN6YPYWWHI6ky1J813KzIq/8+Wg4TO4xU7A/KU= github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.5.2 h1:qLvObTrvO/XRCqmkKxUlOBc48bI3efyDuAZe25QiF0w= -github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8= github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -379,6 +598,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXkPI6mQ4o9DQ0+FKW41hTbunoXZCTqk= github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= @@ -386,8 +607,10 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20180121065927-ffb13db8def0/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -395,6 +618,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/snowflakedb/gosnowflake v1.4.3/go.mod h1:1kyg2XEduwti88V11PKRHImhXLK5WpGiayY6lFNYb98= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -402,7 +626,7 @@ github.com/spf13/afero v1.2.0 h1:O9FblXGxoTc51M+cqr74Bm2Tmt4PvkA5iu/j8HrkNuY= github.com/spf13/afero v1.2.0/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= @@ -413,21 +637,24 @@ github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/y github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/gjson v1.8.1 h1:8j5EE9Hrh3l9Od1OIEDAb7IpezNA20UdRngNAj5N0WU= github.com/tidwall/gjson v1.8.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -446,31 +673,53 @@ github.com/vektra/mockery/v2 v2.2.1 h1:EYgPvxyYkm/0JKs62qlVc9pO+ljb8biPbDWabk5/P github.com/vektra/mockery/v2 v2.2.1/go.mod h1:rBZUbbhMbiSX1WlCGsOgAi6xjuJGxB7KKbnoL0XNYW8= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -478,6 +727,11 @@ golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -490,14 +744,23 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -507,37 +770,72 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190225153610-fe579d43d832/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -545,41 +843,83 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190102155601-82a175fd1598/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190415145633-3fd5a3612ccd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190426135247-a129542de9ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210521090106-6ca3eb03dfc2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -588,50 +928,104 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190425222832-ad9eeb80039a/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190515012406-7d7faa4812bd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200308013534-11ec41452d41/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200827163409-021d7c6f1ec3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200915031644-64986481280e h1:tfSNPIxC48Azhz4nLSPskz/yE9R6ftFRK8pfgfqWUAc= -golang.org/x/tools v0.0.0-20200915031644-64986481280e/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.45.0/go.mod h1:ISLIJCedJolbZvDfAk+Ctuq5hf+aJ33WgtUsfyFoLXA= +google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -639,18 +1033,82 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200911024640-645f7a48b24f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210427215850-f767ed18ee4d/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -662,15 +1120,52 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= +gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= +modernc.org/cc/v3 v3.32.4/go.mod h1:0R6jl1aZlIl2avnYfbfHBS1QB6/f+16mihBObaBC878= +modernc.org/ccgo/v3 v3.9.2/go.mod h1:gnJpy6NIVqkETT+L5zPsQFj7L2kkhfPMzOghRNv/CFo= +modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8= +modernc.org/file v1.0.0/go.mod h1:uqEokAEn1u6e+J45e54dsEA/pw4o7zLrA2GwyntZzjw= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= +modernc.org/libc v1.7.13-0.20210308123627-12f642a52bb8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= +modernc.org/libc v1.9.5/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= +modernc.org/lldb v1.0.0/go.mod h1:jcRvJGWfCGodDZz8BPwiKMJxGJngQ/5DrRapkQnLob8= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/ql v1.0.0/go.mod h1:xGVyrLIatPcO2C1JvI/Co8c0sr6y91HKFNy4pt9JXEY= +modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= +modernc.org/sqlite v1.10.6/go.mod h1:Z9FEjUtZP4qFEg6/SiADg9XCER7aYy9a/j7Pg9P7CPs= +modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/tcl v1.5.2/go.mod h1:pmJYOLgpiys3oI4AeAafkcUfE+TKKilminxNyU/+Zlo= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.0.1-0.20210308123920-1f282aa71362/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= +modernc.org/z v1.0.1/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= +modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sourcegraph.com/sourcegraph/appdash v0.0.0-20180110180208-2cc67fd64755/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= diff --git a/gqlgen.yml b/gqlgen.yml index eab0a4db9..8a21df01b 100644 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -34,24 +34,8 @@ models: model: github.com/stashapp/stash/pkg/models.Movie Tag: model: github.com/stashapp/stash/pkg/models.Tag - ScrapedPerformer: - model: github.com/stashapp/stash/pkg/models.ScrapedPerformer - ScrapedScene: - model: github.com/stashapp/stash/pkg/models.ScrapedScene - ScrapedScenePerformer: - model: github.com/stashapp/stash/pkg/models.ScrapedScenePerformer - ScrapedSceneStudio: - model: github.com/stashapp/stash/pkg/models.ScrapedSceneStudio - ScrapedSceneMovie: - model: github.com/stashapp/stash/pkg/models.ScrapedSceneMovie - ScrapedSceneTag: - model: github.com/stashapp/stash/pkg/models.ScrapedSceneTag SceneFileType: model: github.com/stashapp/stash/pkg/models.SceneFileType - ScrapedMovie: - model: github.com/stashapp/stash/pkg/models.ScrapedMovie - ScrapedMovieStudio: - model: github.com/stashapp/stash/pkg/models.ScrapedMovieStudio SavedFilter: model: github.com/stashapp/stash/pkg/models.SavedFilter StashID: diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index 7912bee00..bb2e41d8a 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -6,6 +6,7 @@ fragment ConfigGeneralData on ConfigGeneralResult { } databasePath generatedPath + metadataPath cachePath calculateMD5 videoFileNamingAlgorithm @@ -18,10 +19,12 @@ fragment ConfigGeneralData on ConfigGeneralResult { previewPreset maxTranscodeSize maxStreamingTranscodeSize + writeImageThumbnails apiKey username password maxSessionAge + trustedProxies logFile logOut logLevel diff --git a/graphql/documents/data/movie-slim.graphql b/graphql/documents/data/movie-slim.graphql index 49f458921..8150986a8 100644 --- a/graphql/documents/data/movie-slim.graphql +++ b/graphql/documents/data/movie-slim.graphql @@ -2,4 +2,4 @@ fragment SlimMovieData on Movie { id name front_image_path -} \ No newline at end of file +} diff --git a/graphql/documents/data/movie.graphql b/graphql/documents/data/movie.graphql index e8e378926..f566e535d 100644 --- a/graphql/documents/data/movie.graphql +++ b/graphql/documents/data/movie.graphql @@ -17,4 +17,10 @@ fragment MovieData on Movie { front_image_path back_image_path scene_count + + scenes { + id + title + path + } } diff --git a/graphql/documents/data/performer.graphql b/graphql/documents/data/performer.graphql index 4c3033c1a..34ff0279d 100644 --- a/graphql/documents/data/performer.graphql +++ b/graphql/documents/data/performer.graphql @@ -22,6 +22,7 @@ fragment PerformerData on Performer { scene_count image_count gallery_count + movie_count tags { ...SlimTagData diff --git a/graphql/documents/data/scene-marker.graphql b/graphql/documents/data/scene-marker.graphql index 30091d857..61439bd1e 100644 --- a/graphql/documents/data/scene-marker.graphql +++ b/graphql/documents/data/scene-marker.graphql @@ -4,6 +4,7 @@ fragment SceneMarkerData on SceneMarker { seconds stream preview + screenshot scene { id diff --git a/graphql/documents/data/scrapers.graphql b/graphql/documents/data/scrapers.graphql index b84fabfc9..7c4632b95 100644 --- a/graphql/documents/data/scrapers.graphql +++ b/graphql/documents/data/scrapers.graphql @@ -1,4 +1,5 @@ fragment ScrapedPerformerData on ScrapedPerformer { + stored_id name gender url @@ -18,7 +19,7 @@ fragment ScrapedPerformerData on ScrapedPerformer { tags { ...ScrapedSceneTagData } - image + images details death_date hair_color @@ -26,7 +27,7 @@ fragment ScrapedPerformerData on ScrapedPerformer { remote_site_id } -fragment ScrapedScenePerformerData on ScrapedScenePerformer { +fragment ScrapedScenePerformerData on ScrapedPerformer { stored_id name gender @@ -55,8 +56,8 @@ fragment ScrapedScenePerformerData on ScrapedScenePerformer { weight } -fragment ScrapedMovieStudioData on ScrapedMovieStudio { - id +fragment ScrapedMovieStudioData on ScrapedStudio { + stored_id name url } @@ -78,7 +79,7 @@ fragment ScrapedMovieData on ScrapedMovie { } } -fragment ScrapedSceneMovieData on ScrapedSceneMovie { +fragment ScrapedSceneMovieData on ScrapedMovie { stored_id name aliases @@ -90,14 +91,14 @@ fragment ScrapedSceneMovieData on ScrapedSceneMovie { synopsis } -fragment ScrapedSceneStudioData on ScrapedSceneStudio { +fragment ScrapedSceneStudioData on ScrapedStudio { stored_id name url remote_site_id } -fragment ScrapedSceneTagData on ScrapedSceneTag { +fragment ScrapedSceneTagData on ScrapedTag { stored_id name } @@ -108,6 +109,7 @@ fragment ScrapedSceneData on ScrapedScene { url date image + remote_site_id file { size @@ -135,6 +137,12 @@ fragment ScrapedSceneData on ScrapedScene { movies { ...ScrapedSceneMovieData } + + fingerprints { + hash + algorithm + duration + } } fragment ScrapedGalleryData on ScrapedGallery { diff --git a/graphql/documents/data/studio-slim.graphql b/graphql/documents/data/studio-slim.graphql index f840ad2fb..36b0fd287 100644 --- a/graphql/documents/data/studio-slim.graphql +++ b/graphql/documents/data/studio-slim.graphql @@ -11,4 +11,5 @@ fragment SlimStudioData on Studio { } details rating + aliases } diff --git a/graphql/documents/data/studio.graphql b/graphql/documents/data/studio.graphql index 68ec86f82..a252ce2f3 100644 --- a/graphql/documents/data/studio.graphql +++ b/graphql/documents/data/studio.graphql @@ -18,10 +18,12 @@ fragment StudioData on Studio { scene_count image_count gallery_count + movie_count stash_ids { stash_id endpoint } details rating + aliases } diff --git a/graphql/documents/data/tag.graphql b/graphql/documents/data/tag.graphql index cf1e2c050..ab83adf73 100644 --- a/graphql/documents/data/tag.graphql +++ b/graphql/documents/data/tag.graphql @@ -8,4 +8,12 @@ fragment TagData on Tag { image_count gallery_count performer_count + + parents { + ...SlimTagData + } + + children { + ...SlimTagData + } } diff --git a/graphql/documents/queries/scrapers/scrapers.graphql b/graphql/documents/queries/scrapers/scrapers.graphql index d5c54bac1..92c0bfd82 100644 --- a/graphql/documents/queries/scrapers/scrapers.graphql +++ b/graphql/documents/queries/scrapers/scrapers.graphql @@ -42,14 +42,14 @@ query ListMovieScrapers { } } -query ScrapePerformerList($scraper_id: ID!, $query: String!) { - scrapePerformerList(scraper_id: $scraper_id, query: $query) { +query ScrapeSinglePerformer($source: ScraperSourceInput!, $input: ScrapeSinglePerformerInput!) { + scrapeSinglePerformer(source: $source, input: $input) { ...ScrapedPerformerData } } -query ScrapePerformer($scraper_id: ID!, $scraped_performer: ScrapedPerformerInput!) { - scrapePerformer(scraper_id: $scraper_id, scraped_performer: $scraped_performer) { +query ScrapeMultiPerformers($source: ScraperSourceInput!, $input: ScrapeMultiPerformersInput!) { + scrapeMultiPerformers(source: $source, input: $input) { ...ScrapedPerformerData } } @@ -60,8 +60,14 @@ query ScrapePerformerURL($url: String!) { } } -query ScrapeScene($scraper_id: ID!, $scene: SceneUpdateInput!) { - scrapeScene(scraper_id: $scraper_id, scene: $scene) { +query ScrapeSingleScene($source: ScraperSourceInput!, $input: ScrapeSingleSceneInput!) { + scrapeSingleScene(source: $source, input: $input) { + ...ScrapedSceneData + } +} + +query ScrapeMultiScenes($source: ScraperSourceInput!, $input: ScrapeMultiScenesInput!) { + scrapeMultiScenes(source: $source, input: $input) { ...ScrapedSceneData } } @@ -72,8 +78,8 @@ query ScrapeSceneURL($url: String!) { } } -query ScrapeGallery($scraper_id: ID!, $gallery: GalleryUpdateInput!) { - scrapeGallery(scraper_id: $scraper_id, gallery: $gallery) { +query ScrapeSingleGallery($source: ScraperSourceInput!, $input: ScrapeSingleGalleryInput!) { + scrapeSingleGallery(source: $source, input: $input) { ...ScrapedGalleryData } } @@ -89,15 +95,3 @@ query ScrapeMovieURL($url: String!) { ...ScrapedMovieData } } - -query QueryStashBoxScene($input: StashBoxSceneQueryInput!) { - queryStashBoxScene(input: $input) { - ...ScrapedStashBoxSceneData - } -} - -query QueryStashBoxPerformer($input: StashBoxPerformerQueryInput!) { - queryStashBoxPerformer(input: $input) { - ...ScrapedStashBoxPerformerData - } -} diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index c0ba269ef..64a55c7eb 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -72,31 +72,50 @@ type Query { listGalleryScrapers: [Scraper!]! listMovieScrapers: [Scraper!]! - """Scrape a list of performers based on name""" - scrapePerformerList(scraper_id: ID!, query: String!): [ScrapedPerformer!]! - """Scrapes a complete performer record based on a scrapePerformerList result""" - scrapePerformer(scraper_id: ID!, scraped_performer: ScrapedPerformerInput!): ScrapedPerformer + """Scrape for a single scene""" + scrapeSingleScene(source: ScraperSourceInput!, input: ScrapeSingleSceneInput!): [ScrapedScene!]! + """Scrape for multiple scenes""" + scrapeMultiScenes(source: ScraperSourceInput!, input: ScrapeMultiScenesInput!): [[ScrapedScene!]!]! + + """Scrape for a single performer""" + scrapeSinglePerformer(source: ScraperSourceInput!, input: ScrapeSinglePerformerInput!): [ScrapedPerformer!]! + """Scrape for multiple performers""" + scrapeMultiPerformers(source: ScraperSourceInput!, input: ScrapeMultiPerformersInput!): [[ScrapedPerformer!]!]! + + """Scrape for a single gallery""" + scrapeSingleGallery(source: ScraperSourceInput!, input: ScrapeSingleGalleryInput!): [ScrapedGallery!]! + + """Scrape for a single movie""" + scrapeSingleMovie(source: ScraperSourceInput!, input: ScrapeSingleMovieInput!): [ScrapedMovie!]! + """Scrapes a complete performer record based on a URL""" scrapePerformerURL(url: String!): ScrapedPerformer - """Scrapes a complete scene record based on an existing scene""" - scrapeScene(scraper_id: ID!, scene: SceneUpdateInput!): ScrapedScene """Scrapes a complete performer record based on a URL""" scrapeSceneURL(url: String!): ScrapedScene - """Scrapes a complete gallery record based on an existing gallery""" - scrapeGallery(scraper_id: ID!, gallery: GalleryUpdateInput!): ScrapedGallery """Scrapes a complete gallery record based on a URL""" scrapeGalleryURL(url: String!): ScrapedGallery """Scrapes a complete movie record based on a URL""" scrapeMovieURL(url: String!): ScrapedMovie + """Scrape a list of performers based on name""" + scrapePerformerList(scraper_id: ID!, query: String!): [ScrapedPerformer!]! @deprecated(reason: "use scrapeSinglePerformer") + """Scrapes a complete performer record based on a scrapePerformerList result""" + scrapePerformer(scraper_id: ID!, scraped_performer: ScrapedPerformerInput!): ScrapedPerformer @deprecated(reason: "use scrapeSinglePerformer") + """Scrapes a complete scene record based on an existing scene""" + scrapeScene(scraper_id: ID!, scene: SceneUpdateInput!): ScrapedScene @deprecated(reason: "use scrapeSingleScene") + """Scrapes a complete gallery record based on an existing gallery""" + scrapeGallery(scraper_id: ID!, gallery: GalleryUpdateInput!): ScrapedGallery @deprecated(reason: "use scrapeSingleGallery") + """Scrape a performer using Freeones""" - scrapeFreeones(performer_name: String!): ScrapedPerformer + scrapeFreeones(performer_name: String!): ScrapedPerformer @deprecated(reason: "use scrapeSinglePerformer with scraper_id = builtin_freeones") """Scrape a list of performers from a query""" - scrapeFreeonesPerformerList(query: String!): [String!]! + scrapeFreeonesPerformerList(query: String!): [String!]! @deprecated(reason: "use scrapeSinglePerformer with scraper_id = builtin_freeones") """Query StashBox for scenes""" - queryStashBoxScene(input: StashBoxSceneQueryInput!): [ScrapedScene!]! - queryStashBoxPerformer(input: StashBoxPerformerQueryInput!): [StashBoxPerformerQueryResult!]! + queryStashBoxScene(input: StashBoxSceneQueryInput!): [ScrapedScene!]! @deprecated(reason: "use scrapeSingleScene or scrapeMultiScenes") + """Query StashBox for performers""" + queryStashBoxPerformer(input: StashBoxPerformerQueryInput!): [StashBoxPerformerQueryResult!]! @deprecated(reason: "use scrapeSinglePerformer or scrapeMultiPerformers") + # === end deprecated methods === # Plugins """List loaded plugins""" diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 63fd730c5..475d4b272 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -39,6 +39,8 @@ input ConfigGeneralInput { databasePath: String """Path to generated files""" generatedPath: String + """Path to import/export files""" + metadataPath: String """Path to cache""" cachePath: String """Whether to calculate MD5 checksums for scene video files""" @@ -63,12 +65,16 @@ input ConfigGeneralInput { maxTranscodeSize: StreamingResolutionEnum """Max streaming transcode size""" maxStreamingTranscodeSize: StreamingResolutionEnum + """Write image thumbnails to disk when generating on the fly""" + writeImageThumbnails: Boolean """Username""" username: String """Password""" password: String """Maximum session cookie age""" maxSessionAge: Int + """Comma separated list of proxies to allow traffic from""" + trustedProxies: [String!] """Name of the log file""" logFile: String """Whether to also output to stderr""" @@ -108,6 +114,8 @@ type ConfigGeneralResult { databasePath: String! """Path to generated files""" generatedPath: String! + """Path to import/export files""" + metadataPath: String! """Path to the config file used""" configFilePath: String! """Path to scrapers""" @@ -136,6 +144,8 @@ type ConfigGeneralResult { maxTranscodeSize: StreamingResolutionEnum """Max streaming transcode size""" maxStreamingTranscodeSize: StreamingResolutionEnum + """Write image thumbnails to disk when generating on the fly""" + writeImageThumbnails: Boolean! """API Key""" apiKey: String! """Username""" @@ -144,6 +154,8 @@ type ConfigGeneralResult { password: String! """Maximum session cookie age""" maxSessionAge: Int! + """Comma separated list of proxies to allow traffic from""" + trustedProxies: [String!]! """Name of the log file""" logFile: String """Whether to also output to stderr""" diff --git a/graphql/schema/types/filters.graphql b/graphql/schema/types/filters.graphql index 989b95863..dd41d5690 100644 --- a/graphql/schema/types/filters.graphql +++ b/graphql/schema/types/filters.graphql @@ -72,7 +72,7 @@ input PerformerFilterType { """Filter to only include performers missing this property""" is_missing: String """Filter to only include performers with these tags""" - tags: MultiCriterionInput + tags: HierarchicalMultiCriterionInput """Filter by tag count""" tag_count: IntCriterionInput """Filter by scene count""" @@ -99,11 +99,11 @@ input PerformerFilterType { input SceneMarkerFilterType { """Filter to only include scene markers with this tag""" - tag_id: ID + tag_id: ID @deprecated(reason: "use tags filter instead") """Filter to only include scene markers with these tags""" - tags: MultiCriterionInput + tags: HierarchicalMultiCriterionInput """Filter to only include scene markers attached to a scene with these tags""" - scene_tags: MultiCriterionInput + scene_tags: HierarchicalMultiCriterionInput """Filter to only include scene markers with these performers""" performers: MultiCriterionInput } @@ -143,11 +143,11 @@ input SceneFilterType { """Filter to only include scenes with this movie""" movies: MultiCriterionInput """Filter to only include scenes with these tags""" - tags: MultiCriterionInput + tags: HierarchicalMultiCriterionInput """Filter by tag count""" tag_count: IntCriterionInput """Filter to only include scenes with performers with these tags""" - performer_tags: MultiCriterionInput + performer_tags: HierarchicalMultiCriterionInput """Filter to only include scenes with these performers""" performers: MultiCriterionInput """Filter by performer count""" @@ -176,9 +176,15 @@ input MovieFilterType { is_missing: String """Filter by url""" url: StringCriterionInput + """Filter to only include movies where performer appears in a scene""" + performers: MultiCriterionInput } input StudioFilterType { + AND: StudioFilterType + OR: StudioFilterType + NOT: StudioFilterType + name: StringCriterionInput details: StringCriterionInput """Filter to only include studios with this parent studio""" @@ -197,6 +203,8 @@ input StudioFilterType { gallery_count: IntCriterionInput """Filter by url""" url: StringCriterionInput + """Filter by studio aliases""" + aliases: StringCriterionInput } input GalleryFilterType { @@ -224,11 +232,11 @@ input GalleryFilterType { """Filter to only include galleries with this studio""" studios: HierarchicalMultiCriterionInput """Filter to only include galleries with these tags""" - tags: MultiCriterionInput + tags: HierarchicalMultiCriterionInput """Filter by tag count""" tag_count: IntCriterionInput """Filter to only include galleries with performers with these tags""" - performer_tags: MultiCriterionInput + performer_tags: HierarchicalMultiCriterionInput """Filter to only include galleries with these performers""" performers: MultiCriterionInput """Filter by performer count""" @@ -267,6 +275,18 @@ input TagFilterType { """Filter by number of markers with this tag""" marker_count: IntCriterionInput + + """Filter by parent tags""" + parents: HierarchicalMultiCriterionInput + + """Filter by child tags""" + children: HierarchicalMultiCriterionInput + + """Filter by number of parent tags the tag has""" + parent_count: IntCriterionInput + + """Filter by number f child tags the tag has""" + child_count: IntCriterionInput } input ImageFilterType { @@ -293,11 +313,11 @@ input ImageFilterType { """Filter to only include images with this studio""" studios: HierarchicalMultiCriterionInput """Filter to only include images with these tags""" - tags: MultiCriterionInput + tags: HierarchicalMultiCriterionInput """Filter by tag count""" tag_count: IntCriterionInput """Filter to only include images with performers with these tags""" - performer_tags: MultiCriterionInput + performer_tags: HierarchicalMultiCriterionInput """Filter to only include images with these performers""" performers: MultiCriterionInput """Filter by performer count""" @@ -357,7 +377,7 @@ input GenderCriterionInput { input HierarchicalMultiCriterionInput { value: [ID!] modifier: CriterionModifier! - depth: Int! + depth: Int } enum FilterMode { diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index 567b77079..cfc366ccc 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -1,13 +1,15 @@ scalar Upload input GenerateMetadataInput { - sprites: Boolean! - previews: Boolean! - imagePreviews: Boolean! + sprites: Boolean + previews: Boolean + imagePreviews: Boolean previewOptions: GeneratePreviewOptionsInput - markers: Boolean! - transcodes: Boolean! - phashes: Boolean! + markers: Boolean + markerImagePreviews: Boolean + markerScreenshots: Boolean + transcodes: Boolean + phashes: Boolean """scene ids to generate for""" sceneIDs: [ID!] @@ -45,6 +47,8 @@ input ScanMetadataInput { scanGenerateSprites: Boolean """Generate phashes during scan""" scanGeneratePhashes: Boolean + """Generate image thumbnails during scan""" + scanGenerateThumbnails: Boolean } input CleanMetadataInput { diff --git a/graphql/schema/types/movie.graphql b/graphql/schema/types/movie.graphql index 3fb156311..104c68176 100644 --- a/graphql/schema/types/movie.graphql +++ b/graphql/schema/types/movie.graphql @@ -17,6 +17,7 @@ type Movie { front_image_path: String # Resolver back_image_path: String # Resolver scene_count: Int # Resolver + scenes: [Scene!]! } input MovieCreateInput { @@ -60,4 +61,4 @@ input MovieDestroyInput { type FindMoviesResultType { count: Int! movies: [Movie!]! -} \ No newline at end of file +} diff --git a/graphql/schema/types/performer.graphql b/graphql/schema/types/performer.graphql index 35c9eb36d..8c0c6e396 100644 --- a/graphql/schema/types/performer.graphql +++ b/graphql/schema/types/performer.graphql @@ -42,6 +42,8 @@ type Performer { weight: Int created_at: Time! updated_at: Time! + movie_count: Int + movies: [Movie!]! } input PerformerCreateInput { diff --git a/graphql/schema/types/scene-marker.graphql b/graphql/schema/types/scene-marker.graphql index f29380f26..8e3e54c81 100644 --- a/graphql/schema/types/scene-marker.graphql +++ b/graphql/schema/types/scene-marker.graphql @@ -12,6 +12,8 @@ type SceneMarker { stream: String! # Resolver """The path to the preview image for this marker""" preview: String! # Resolver + """The path to the screenshot image for this marker""" + screenshot: String! # Resolver } input SceneMarkerCreateInput { diff --git a/graphql/schema/types/scene.graphql b/graphql/schema/types/scene.graphql index 734b5f596..4e2b0281b 100644 --- a/graphql/schema/types/scene.graphql +++ b/graphql/schema/types/scene.graphql @@ -103,6 +103,7 @@ input BulkSceneUpdateInput { gallery_ids: BulkUpdateIds performer_ids: BulkUpdateIds tag_ids: BulkUpdateIds + movie_ids: BulkUpdateIds } input SceneDestroyInput { diff --git a/graphql/schema/types/scraped-movie.graphql b/graphql/schema/types/scraped-movie.graphql index d1546dfb9..55efb693d 100644 --- a/graphql/schema/types/scraped-movie.graphql +++ b/graphql/schema/types/scraped-movie.graphql @@ -1,12 +1,6 @@ -type ScrapedMovieStudio { - """Set if studio matched""" - id: ID - name: String! - url: String -} - """A movie from a scraping operation...""" type ScrapedMovie { + stored_id: ID name: String aliases: String duration: String @@ -15,7 +9,7 @@ type ScrapedMovie { director: String url: String synopsis: String - studio: ScrapedMovieStudio + studio: ScrapedStudio """This should be a base64 encoded data URL""" front_image: String diff --git a/graphql/schema/types/scraped-performer.graphql b/graphql/schema/types/scraped-performer.graphql index 2ae1b5a8a..b11b9b1b5 100644 --- a/graphql/schema/types/scraped-performer.graphql +++ b/graphql/schema/types/scraped-performer.graphql @@ -1,5 +1,7 @@ """A performer from a scraping operation...""" type ScrapedPerformer { + """Set if performer matched""" + stored_id: ID name: String gender: String url: String @@ -16,11 +18,11 @@ type ScrapedPerformer { tattoos: String piercings: String aliases: String - # Should be ScrapedPerformerTag - but would be identical types - tags: [ScrapedSceneTag!] + tags: [ScrapedTag!] """This should be a base64 encoded data URL""" - image: String + image: String @deprecated(reason: "use images instead") + images: [String!] details: String death_date: String hair_color: String @@ -29,6 +31,8 @@ type ScrapedPerformer { } input ScrapedPerformerInput { + """Set if performer matched""" + stored_id: ID name: String gender: String url: String diff --git a/graphql/schema/types/scraper.graphql b/graphql/schema/types/scraper.graphql index 860457bb0..9e35346f4 100644 --- a/graphql/schema/types/scraper.graphql +++ b/graphql/schema/types/scraper.graphql @@ -26,49 +26,7 @@ type Scraper { movie: ScraperSpec } -type ScrapedScenePerformer { - """Set if performer matched""" - stored_id: ID - name: String! - gender: String - url: String - twitter: String - instagram: String - birthdate: String - ethnicity: String - country: String - eye_color: String - height: String - measurements: String - fake_tits: String - career_length: String - tattoos: String - piercings: String - aliases: String - tags: [ScrapedSceneTag!] - - remote_site_id: String - images: [String!] - details: String - death_date: String - hair_color: String - weight: String -} - -type ScrapedSceneMovie { - """Set if movie matched""" - stored_id: ID - name: String! - aliases: String - duration: String - date: String - rating: String - director: String - synopsis: String - url: String -} - -type ScrapedSceneStudio { +type ScrapedStudio { """Set if studio matched""" stored_id: ID name: String! @@ -77,7 +35,7 @@ type ScrapedSceneStudio { remote_site_id: String } -type ScrapedSceneTag { +type ScrapedTag { """Set if tag matched""" stored_id: ID name: String! @@ -94,25 +52,98 @@ type ScrapedScene { file: SceneFileType # Resolver - studio: ScrapedSceneStudio - tags: [ScrapedSceneTag!] - performers: [ScrapedScenePerformer!] - movies: [ScrapedSceneMovie!] + studio: ScrapedStudio + tags: [ScrapedTag!] + performers: [ScrapedPerformer!] + movies: [ScrapedMovie!] remote_site_id: String duration: Int fingerprints: [StashBoxFingerprint!] } +input ScrapedSceneInput { + title: String + details: String + url: String + date: String + + # no image, file, duration or relationships + + remote_site_id: String +} + type ScrapedGallery { title: String details: String url: String date: String - studio: ScrapedSceneStudio - tags: [ScrapedSceneTag!] - performers: [ScrapedScenePerformer!] + studio: ScrapedStudio + tags: [ScrapedTag!] + performers: [ScrapedPerformer!] +} + +input ScrapedGalleryInput { + title: String + details: String + url: String + date: String + + # no studio, tags or performers +} + +input ScraperSourceInput { + """Index of the configured stash-box instance to use. Should be unset if scraper_id is set""" + stash_box_index: Int + """Scraper ID to scrape with. Should be unset if stash_box_index is set""" + scraper_id: ID +} + +input ScrapeSingleSceneInput { + """Instructs to query by string""" + query: String + """Instructs to query by scene fingerprints""" + scene_id: ID + """Instructs to query by scene fragment""" + scene_input: ScrapedSceneInput +} + +input ScrapeMultiScenesInput { + """Instructs to query by scene fingerprints""" + scene_ids: [ID!] +} + +input ScrapeSinglePerformerInput { + """Instructs to query by string""" + query: String + """Instructs to query by performer id""" + performer_id: ID + """Instructs to query by performer fragment""" + performer_input: ScrapedPerformerInput +} + +input ScrapeMultiPerformersInput { + """Instructs to query by scene fingerprints""" + performer_ids: [ID!] +} + +input ScrapeSingleGalleryInput { + """Instructs to query by string""" + query: String + """Instructs to query by gallery id""" + gallery_id: ID + """Instructs to query by gallery fragment""" + gallery_input: ScrapedGalleryInput +} + +input ScrapeSingleMovieInput { + """Instructs to query by string""" + query: String + """Instructs to query by movie id""" + movie_id: ID + """Instructs to query by gallery fragment""" + movie_input: ScrapedMovieInput } input StashBoxSceneQueryInput { @@ -135,7 +166,7 @@ input StashBoxPerformerQueryInput { type StashBoxPerformerQueryResult { query: String! - results: [ScrapedScenePerformer!]! + results: [ScrapedPerformer!]! } type StashBoxFingerprint { diff --git a/graphql/schema/types/studio.graphql b/graphql/schema/types/studio.graphql index 364b7ad42..183ffc5f6 100644 --- a/graphql/schema/types/studio.graphql +++ b/graphql/schema/types/studio.graphql @@ -5,6 +5,7 @@ type Studio { url: String parent_studio: Studio child_studios: [Studio!]! + aliases: [String!]! image_path: String # Resolver scene_count: Int # Resolver @@ -15,6 +16,8 @@ type Studio { details: String created_at: Time! updated_at: Time! + movie_count: Int + movies: [Movie!]! } input StudioCreateInput { @@ -26,6 +29,7 @@ input StudioCreateInput { stash_ids: [StashIDInput!] rating: Int details: String + aliases: [String!] } input StudioUpdateInput { @@ -38,6 +42,7 @@ input StudioUpdateInput { stash_ids: [StashIDInput!] rating: Int details: String + aliases: [String!] } input StudioDestroyInput { diff --git a/graphql/schema/types/tag.graphql b/graphql/schema/types/tag.graphql index 46f8c25cd..fea9f40fe 100644 --- a/graphql/schema/types/tag.graphql +++ b/graphql/schema/types/tag.graphql @@ -11,6 +11,9 @@ type Tag { image_count: Int # Resolver gallery_count: Int # Resolver performer_count: Int + + parents: [Tag!]! + children: [Tag!]! } input TagCreateInput { @@ -19,6 +22,9 @@ input TagCreateInput { """This should be a URL or a base64 encoded data URL""" image: String + + parent_ids: [ID!] + child_ids: [ID!] } input TagUpdateInput { @@ -28,6 +34,9 @@ input TagUpdateInput { """This should be a URL or a base64 encoded data URL""" image: String + + parent_ids: [ID!] + child_ids: [ID!] } input TagDestroyInput { diff --git a/main.go b/main.go index e08271a7f..6d59d8085 100644 --- a/main.go +++ b/main.go @@ -2,25 +2,38 @@ package main import ( + "embed" "os" "os/signal" "runtime/pprof" "syscall" "github.com/stashapp/stash/pkg/api" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager" _ "github.com/golang-migrate/migrate/v4/database/sqlite3" _ "github.com/golang-migrate/migrate/v4/source/file" ) +//go:embed ui/v2.5/build +var uiBox embed.FS + +//go:embed ui/login +var loginUIBox embed.FS + func main() { manager.Initialize() - api.Start() + api.Start(uiBox, loginUIBox) // stop any profiling at exit defer pprof.StopCPUProfile() blockForever() + + err := manager.GetInstance().Shutdown() + if err != nil { + logger.Errorf("Error when closing: %s", err) + } } func blockForever() { diff --git a/pkg/api/authentication.go b/pkg/api/authentication.go new file mode 100644 index 000000000..ae6fb2952 --- /dev/null +++ b/pkg/api/authentication.go @@ -0,0 +1,140 @@ +package api + +import ( + "net" + "net/http" + "net/url" + "strings" + + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/pkg/session" +) + +const loginEndPoint = "/login" + +const ( + tripwireActivatedErrMsg = "Stash is exposed to the public internet without authentication, and is not serving any more content to protect your privacy. " + + "More information and fixes are available at https://github.com/stashapp/stash/wiki/Authentication-Required-When-Accessing-Stash-From-the-Internet" + + externalAccessErrMsg = "You have attempted to access Stash over the internet, and authentication is not enabled. " + + "This is extremely dangerous! The whole world can see your your stash page and browse your files! " + + "Stash is not answering any other requests to protect your privacy. " + + "Please read the log entry or visit https://github.com/stashapp/stash/wiki/Authentication-Required-When-Accessing-Stash-From-the-Internet" +) + +func allowUnauthenticated(r *http.Request) bool { + return strings.HasPrefix(r.URL.Path, loginEndPoint) || r.URL.Path == "/css" +} + +func authenticateHandler() func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c := config.GetInstance() + + if !checkSecurityTripwireActivated(c, w) { + return + } + + userID, err := manager.GetInstance().SessionStore.Authenticate(w, r) + if err != nil { + if err != session.ErrUnauthorized { + w.WriteHeader(http.StatusInternalServerError) + _, err = w.Write([]byte(err.Error())) + if err != nil { + logger.Error(err) + } + return + } + + // unauthorized error + w.Header().Add("WWW-Authenticate", `FormBased`) + w.WriteHeader(http.StatusUnauthorized) + return + } + + if err := session.CheckAllowPublicWithoutAuth(c, r); err != nil { + switch err := err.(type) { + case session.ExternalAccessError: + securityActivateTripwireAccessedFromInternetWithoutAuth(c, err, w) + return + case session.UntrustedProxyError: + logger.Warnf("Rejected request from untrusted proxy: %s", net.IP(err).String()) + w.WriteHeader(http.StatusForbidden) + return + default: + logger.Errorf("Error checking external access security: %s", err.Error()) + w.WriteHeader(http.StatusInternalServerError) + return + } + } + + ctx := r.Context() + + if c.HasCredentials() { + // authentication is required + if userID == "" && !allowUnauthenticated(r) { + // authentication was not received, redirect + // if graphql was requested, we just return a forbidden error + if r.URL.Path == "/graphql" { + w.Header().Add("WWW-Authenticate", `FormBased`) + w.WriteHeader(http.StatusUnauthorized) + return + } + + prefix := getProxyPrefix(r.Header) + + // otherwise redirect to the login page + u := url.URL{ + Path: prefix + "/login", + } + q := u.Query() + q.Set(returnURLParam, prefix+r.URL.Path) + u.RawQuery = q.Encode() + http.Redirect(w, r, u.String(), http.StatusFound) + return + } + } + + ctx = session.SetCurrentUserID(ctx, userID) + + r = r.WithContext(ctx) + + next.ServeHTTP(w, r) + }) + } +} + +func checkSecurityTripwireActivated(c *config.Instance, w http.ResponseWriter) bool { + if accessErr := session.CheckExternalAccessTripwire(c); accessErr != nil { + w.WriteHeader(http.StatusForbidden) + _, err := w.Write([]byte(tripwireActivatedErrMsg)) + if err != nil { + logger.Error(err) + } + return false + } + + return true +} + +func securityActivateTripwireAccessedFromInternetWithoutAuth(c *config.Instance, accessErr session.ExternalAccessError, w http.ResponseWriter) { + session.LogExternalAccessError(accessErr) + + err := c.ActivatePublicAccessTripwire(net.IP(accessErr).String()) + if err != nil { + logger.Error(err) + } + + w.WriteHeader(http.StatusForbidden) + _, err = w.Write([]byte(externalAccessErrMsg)) + if err != nil { + logger.Error(err) + } + + err = manager.GetInstance().Shutdown() + if err != nil { + logger.Error(err) + } +} diff --git a/pkg/api/changeset_translator.go b/pkg/api/changeset_translator.go index 3864b1082..e1fc3868a 100644 --- a/pkg/api/changeset_translator.go +++ b/pkg/api/changeset_translator.go @@ -20,7 +20,7 @@ func getArgumentMap(ctx context.Context) map[string]interface{} { func getUpdateInputMap(ctx context.Context) map[string]interface{} { args := getArgumentMap(ctx) - input, _ := args[updateInputField] + input := args[updateInputField] var ret map[string]interface{} if input != nil { ret, _ = input.(map[string]interface{}) @@ -36,7 +36,7 @@ func getUpdateInputMap(ctx context.Context) map[string]interface{} { func getUpdateInputMaps(ctx context.Context) []map[string]interface{} { args := getArgumentMap(ctx) - input, _ := args[updateInputField] + input := args[updateInputField] var ret []map[string]interface{} if input != nil { // convert []interface{} into []map[string]interface{} diff --git a/pkg/api/check_version.go b/pkg/api/check_version.go index 659e01ec7..44e29c52a 100644 --- a/pkg/api/check_version.go +++ b/pkg/api/check_version.go @@ -4,7 +4,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "regexp" "runtime" @@ -29,6 +29,7 @@ var ErrNoVersion = errors.New("no stash version") var stashReleases = func() map[string]string { return map[string]string{ "darwin/amd64": "stash-osx", + "darwin/arm64": "stash-osx-applesilicon", "linux/amd64": "stash-linux", "windows/amd64": "stash-win.exe", "linux/arm": "stash-pi", @@ -117,23 +118,26 @@ func makeGithubRequest(url string, output interface{}) error { response, err := client.Do(req) if err != nil { + //lint:ignore ST1005 Github is a proper capitalized noun return fmt.Errorf("Github API request failed: %s", err) } if response.StatusCode != http.StatusOK { + //lint:ignore ST1005 Github is a proper capitalized noun return fmt.Errorf("Github API request failed: %s", response.Status) } defer response.Body.Close() - data, err := ioutil.ReadAll(response.Body) + data, err := io.ReadAll(response.Body) if err != nil { + //lint:ignore ST1005 Github is a proper capitalized noun return fmt.Errorf("Github API read response failed: %s", err) } err = json.Unmarshal(data, output) if err != nil { - return fmt.Errorf("Unmarshalling Github API response failed: %s", err) + return fmt.Errorf("unmarshalling Github API response failed: %s", err) } return nil @@ -196,7 +200,7 @@ func GetLatestVersion(shortHash bool) (latestVersion string, latestRelease strin } if latestVersion == "" { - return "", "", fmt.Errorf("No version found for \"%s\"", version) + return "", "", fmt.Errorf("no version found for \"%s\"", version) } return latestVersion, latestRelease, nil } diff --git a/pkg/api/context_keys.go b/pkg/api/context_keys.go index 839464af9..8731f75c3 100644 --- a/pkg/api/context_keys.go +++ b/pkg/api/context_keys.go @@ -5,8 +5,8 @@ package api type key int const ( - galleryKey key = iota - performerKey + // galleryKey key = 0 + performerKey key = iota + 1 sceneKey studioKey movieKey diff --git a/pkg/api/images.go b/pkg/api/images.go index e38fb6aeb..140478a7a 100644 --- a/pkg/api/images.go +++ b/pkg/api/images.go @@ -1,24 +1,36 @@ package api import ( + "io" + "io/fs" + "os" "strings" - "github.com/gobuffalo/packr/v2" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/pkg/static" "github.com/stashapp/stash/pkg/utils" ) type imageBox struct { - box *packr.Box + box fs.FS files []string } -func newImageBox(box *packr.Box) *imageBox { - return &imageBox{ - box: box, - files: box.List(), +func newImageBox(box fs.FS) (*imageBox, error) { + ret := &imageBox{ + box: box, } + + err := fs.WalkDir(box, ".", func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { + ret.files = append(ret.files, path) + } + + return nil + }) + + return ret, err } var performerBox *imageBox @@ -26,8 +38,15 @@ var performerBoxMale *imageBox var performerBoxCustom *imageBox func initialiseImages() { - performerBox = newImageBox(packr.New("Performer Box", "../../static/performer")) - performerBoxMale = newImageBox(packr.New("Male Performer Box", "../../static/performer_male")) + var err error + performerBox, err = newImageBox(&static.Performer) + if err != nil { + logger.Warnf("error loading performer images: %v", err) + } + performerBoxMale, err = newImageBox(&static.PerformerMale) + if err != nil { + logger.Warnf("error loading male performer images: %v", err) + } initialiseCustomImages() } @@ -36,7 +55,11 @@ func initialiseCustomImages() { if customPath != "" { logger.Debugf("Loading custom performer images from %s", customPath) // We need to set performerBoxCustom at runtime, as this is a custom path, and store it in a pointer. - performerBoxCustom = newImageBox(packr.Folder(customPath)) + var err error + performerBoxCustom, err = newImageBox(os.DirFS(customPath)) + if err != nil { + logger.Warnf("error loading custom performer from %s: %v", customPath, err) + } } else { performerBoxCustom = nil } @@ -63,5 +86,11 @@ func getRandomPerformerImageUsingName(name, gender, customPath string) ([]byte, imageFiles := box.files index := utils.IntFromString(name) % uint64(len(imageFiles)) - return box.box.Find(imageFiles[index]) + img, err := box.box.Open(imageFiles[index]) + if err != nil { + return nil, err + } + defer img.Close() + + return io.ReadAll(img) } diff --git a/pkg/api/resolver.go b/pkg/api/resolver.go index ed15a1d21..7d3f6aa3f 100644 --- a/pkg/api/resolver.go +++ b/pkg/api/resolver.go @@ -53,22 +53,6 @@ func (r *Resolver) Tag() models.TagResolver { return &tagResolver{r} } -func (r *Resolver) ScrapedSceneTag() models.ScrapedSceneTagResolver { - return &scrapedSceneTagResolver{r} -} - -func (r *Resolver) ScrapedSceneMovie() models.ScrapedSceneMovieResolver { - return &scrapedSceneMovieResolver{r} -} - -func (r *Resolver) ScrapedScenePerformer() models.ScrapedScenePerformerResolver { - return &scrapedScenePerformerResolver{r} -} - -func (r *Resolver) ScrapedSceneStudio() models.ScrapedSceneStudioResolver { - return &scrapedSceneStudioResolver{r} -} - type mutationResolver struct{ *Resolver } type queryResolver struct{ *Resolver } type subscriptionResolver struct{ *Resolver } @@ -81,10 +65,6 @@ type imageResolver struct{ *Resolver } type studioResolver struct{ *Resolver } type movieResolver struct{ *Resolver } type tagResolver struct{ *Resolver } -type scrapedSceneTagResolver struct{ *Resolver } -type scrapedSceneMovieResolver struct{ *Resolver } -type scrapedScenePerformerResolver struct{ *Resolver } -type scrapedSceneStudioResolver struct{ *Resolver } func (r *Resolver) withTxn(ctx context.Context, fn func(r models.Repository) error) error { return r.txnManager.WithTxn(ctx, fn) diff --git a/pkg/api/resolver_model_movie.go b/pkg/api/resolver_model_movie.go index 6bfa759f7..0b3597d49 100644 --- a/pkg/api/resolver_model_movie.go +++ b/pkg/api/resolver_model_movie.go @@ -125,6 +125,18 @@ func (r *movieResolver) SceneCount(ctx context.Context, obj *models.Movie) (ret return &res, err } +func (r *movieResolver) Scenes(ctx context.Context, obj *models.Movie) (ret []*models.Scene, err error) { + if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { + var err error + ret, err = repo.Scene().FindByMovieID(obj.ID) + return err + }); err != nil { + return nil, err + } + + return ret, nil +} + func (r *movieResolver) CreatedAt(ctx context.Context, obj *models.Movie) (*time.Time, error) { return &obj.CreatedAt.Timestamp, nil } diff --git a/pkg/api/resolver_model_performer.go b/pkg/api/resolver_model_performer.go index d27cce38b..ea52873df 100644 --- a/pkg/api/resolver_model_performer.go +++ b/pkg/api/resolver_model_performer.go @@ -254,3 +254,26 @@ func (r *performerResolver) CreatedAt(ctx context.Context, obj *models.Performer func (r *performerResolver) UpdatedAt(ctx context.Context, obj *models.Performer) (*time.Time, error) { return &obj.UpdatedAt.Timestamp, nil } + +func (r *performerResolver) Movies(ctx context.Context, obj *models.Performer) (ret []*models.Movie, err error) { + if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { + ret, err = repo.Movie().FindByPerformerID(obj.ID) + return err + }); err != nil { + return nil, err + } + + return ret, nil +} + +func (r *performerResolver) MovieCount(ctx context.Context, obj *models.Performer) (ret *int, err error) { + var res int + if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { + res, err = repo.Movie().CountByPerformerID(obj.ID) + return err + }); err != nil { + return nil, err + } + + return &res, nil +} diff --git a/pkg/api/resolver_model_scene_marker.go b/pkg/api/resolver_model_scene_marker.go index 84bd5c9f2..3a6ada5bc 100644 --- a/pkg/api/resolver_model_scene_marker.go +++ b/pkg/api/resolver_model_scene_marker.go @@ -58,6 +58,12 @@ func (r *sceneMarkerResolver) Preview(ctx context.Context, obj *models.SceneMark return urlbuilders.NewSceneURLBuilder(baseURL, sceneID).GetSceneMarkerStreamPreviewURL(obj.ID), nil } +func (r *sceneMarkerResolver) Screenshot(ctx context.Context, obj *models.SceneMarker) (string, error) { + baseURL, _ := ctx.Value(BaseURLCtxKey).(string) + sceneID := int(obj.SceneID.Int64) + return urlbuilders.NewSceneURLBuilder(baseURL, sceneID).GetSceneMarkerStreamScreenshotURL(obj.ID), nil +} + func (r *sceneMarkerResolver) CreatedAt(ctx context.Context, obj *models.SceneMarker) (*time.Time, error) { return &obj.CreatedAt.Timestamp, nil } diff --git a/pkg/api/resolver_model_scraper.go b/pkg/api/resolver_model_scraper.go deleted file mode 100644 index 583194496..000000000 --- a/pkg/api/resolver_model_scraper.go +++ /dev/null @@ -1,23 +0,0 @@ -package api - -import ( - "context" - - "github.com/stashapp/stash/pkg/models" -) - -func (r *scrapedSceneTagResolver) StoredID(ctx context.Context, obj *models.ScrapedSceneTag) (*string, error) { - return obj.ID, nil -} - -func (r *scrapedSceneMovieResolver) StoredID(ctx context.Context, obj *models.ScrapedSceneMovie) (*string, error) { - return obj.ID, nil -} - -func (r *scrapedScenePerformerResolver) StoredID(ctx context.Context, obj *models.ScrapedScenePerformer) (*string, error) { - return obj.ID, nil -} - -func (r *scrapedSceneStudioResolver) StoredID(ctx context.Context, obj *models.ScrapedSceneStudio) (*string, error) { - return obj.ID, nil -} diff --git a/pkg/api/resolver_model_studio.go b/pkg/api/resolver_model_studio.go index 5123fe8b1..e8724c545 100644 --- a/pkg/api/resolver_model_studio.go +++ b/pkg/api/resolver_model_studio.go @@ -45,6 +45,17 @@ func (r *studioResolver) ImagePath(ctx context.Context, obj *models.Studio) (*st return &imagePath, nil } +func (r *studioResolver) Aliases(ctx context.Context, obj *models.Studio) (ret []string, err error) { + if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { + ret, err = repo.Studio().GetAliases(obj.ID) + return err + }); err != nil { + return nil, err + } + + return ret, err +} + func (r *studioResolver) SceneCount(ctx context.Context, obj *models.Studio) (ret *int, err error) { var res int if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { @@ -140,3 +151,26 @@ func (r *studioResolver) CreatedAt(ctx context.Context, obj *models.Studio) (*ti func (r *studioResolver) UpdatedAt(ctx context.Context, obj *models.Studio) (*time.Time, error) { return &obj.UpdatedAt.Timestamp, nil } + +func (r *studioResolver) Movies(ctx context.Context, obj *models.Studio) (ret []*models.Movie, err error) { + if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { + ret, err = repo.Movie().FindByStudioID(obj.ID) + return err + }); err != nil { + return nil, err + } + + return ret, nil +} + +func (r *studioResolver) MovieCount(ctx context.Context, obj *models.Studio) (ret *int, err error) { + var res int + if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { + res, err = repo.Movie().CountByStudioID(obj.ID) + return err + }); err != nil { + return nil, err + } + + return &res, nil +} diff --git a/pkg/api/resolver_model_tag.go b/pkg/api/resolver_model_tag.go index a863cd233..3b90463ca 100644 --- a/pkg/api/resolver_model_tag.go +++ b/pkg/api/resolver_model_tag.go @@ -10,6 +10,28 @@ import ( "github.com/stashapp/stash/pkg/models" ) +func (r *tagResolver) Parents(ctx context.Context, obj *models.Tag) (ret []*models.Tag, err error) { + if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { + ret, err = repo.Tag().FindByChildTagID(obj.ID) + return err + }); err != nil { + return nil, err + } + + return ret, nil +} + +func (r *tagResolver) Children(ctx context.Context, obj *models.Tag) (ret []*models.Tag, err error) { + if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { + ret, err = repo.Tag().FindByParentTagID(obj.ID) + return err + }); err != nil { + return nil, err + } + + return ret, nil +} + func (r *tagResolver) Aliases(ctx context.Context, obj *models.Tag) (ret []string, err error) { if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { ret, err = repo.Tag().GetAliases(obj.ID) diff --git a/pkg/api/resolver_mutation_configure.go b/pkg/api/resolver_mutation_configure.go index 4eb477731..6672a1993 100644 --- a/pkg/api/resolver_mutation_configure.go +++ b/pkg/api/resolver_mutation_configure.go @@ -61,6 +61,15 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co c.Set(config.Generated, input.GeneratedPath) } + if input.MetadataPath != nil { + if *input.MetadataPath != "" { + if err := utils.EnsureDir(*input.MetadataPath); err != nil { + return makeConfigGeneralResult(), err + } + } + c.Set(config.Metadata, input.MetadataPath) + } + if input.CachePath != nil { if *input.CachePath != "" { if err := utils.EnsureDir(*input.CachePath); err != nil { @@ -115,6 +124,10 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co c.Set(config.MaxStreamingTranscodeSize, input.MaxStreamingTranscodeSize.String()) } + if input.WriteImageThumbnails != nil { + c.Set(config.WriteImageThumbnails, *input.WriteImageThumbnails) + } + if input.Username != nil { c.Set(config.Username, input.Username) } @@ -133,6 +146,10 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co c.Set(config.MaxSessionAge, *input.MaxSessionAge) } + if input.TrustedProxies != nil { + c.Set(config.TrustedProxies, input.TrustedProxies) + } + if input.LogFile != nil { c.Set(config.LogFile, input.LogFile) } @@ -289,7 +306,9 @@ func (r *mutationResolver) ConfigureDlna(ctx context.Context, input models.Confi if !*input.Enabled && dlnaService.IsRunning() { dlnaService.Stop(nil) } else if *input.Enabled && !dlnaService.IsRunning() { - dlnaService.Start(nil) + if err := dlnaService.Start(nil); err != nil { + logger.Warnf("error starting DLNA service: %v", err) + } } } diff --git a/pkg/api/resolver_mutation_metadata.go b/pkg/api/resolver_mutation_metadata.go index b43e1b224..aee27005e 100644 --- a/pkg/api/resolver_mutation_metadata.go +++ b/pkg/api/resolver_mutation_metadata.go @@ -2,7 +2,8 @@ package api import ( "context" - "io/ioutil" + "fmt" + "os" "path/filepath" "strconv" "sync" @@ -105,8 +106,10 @@ func (r *mutationResolver) BackupDatabase(ctx context.Context, input models.Back mgr := manager.GetInstance() var backupPath string if download { - utils.EnsureDir(mgr.Paths.Generated.Downloads) - f, err := ioutil.TempFile(mgr.Paths.Generated.Downloads, "backup*.sqlite") + if err := utils.EnsureDir(mgr.Paths.Generated.Downloads); err != nil { + return nil, fmt.Errorf("could not create backup directory %v: %w", mgr.Paths.Generated.Downloads, err) + } + f, err := os.CreateTemp(mgr.Paths.Generated.Downloads, "backup*.sqlite") if err != nil { return nil, err } diff --git a/pkg/api/resolver_mutation_plugin.go b/pkg/api/resolver_mutation_plugin.go index 832f21371..656356ff7 100644 --- a/pkg/api/resolver_mutation_plugin.go +++ b/pkg/api/resolver_mutation_plugin.go @@ -17,7 +17,7 @@ func (r *mutationResolver) RunPluginTask(ctx context.Context, pluginID string, t func (r *mutationResolver) ReloadPlugins(ctx context.Context) (bool, error) { err := manager.GetInstance().PluginCache.LoadPlugins() if err != nil { - logger.Errorf("Error reading plugin configs: %s", err.Error()) + logger.Errorf("Error reading plugin configs: %v", err) } return true, nil diff --git a/pkg/api/resolver_mutation_scene.go b/pkg/api/resolver_mutation_scene.go index a0e788454..090599665 100644 --- a/pkg/api/resolver_mutation_scene.go +++ b/pkg/api/resolver_mutation_scene.go @@ -304,6 +304,18 @@ func (r *mutationResolver) BulkSceneUpdate(ctx context.Context, input models.Bul return err } } + + // Save the movies + if translator.hasField("movie_ids") { + movies, err := adjustSceneMovieIDs(qb, sceneID, *input.MovieIds) + if err != nil { + return err + } + + if err := qb.UpdateMovies(sceneID, movies); err != nil { + return err + } + } } return nil @@ -395,6 +407,48 @@ func adjustSceneGalleryIDs(qb models.SceneReader, sceneID int, ids models.BulkUp return adjustIDs(ret, ids), nil } +func adjustSceneMovieIDs(qb models.SceneReader, sceneID int, updateIDs models.BulkUpdateIds) ([]models.MoviesScenes, error) { + existingMovies, err := qb.GetMovies(sceneID) + if err != nil { + return nil, err + } + + // if we are setting the ids, just return the ids + if updateIDs.Mode == models.BulkUpdateIDModeSet { + existingMovies = []models.MoviesScenes{} + for _, idStr := range updateIDs.Ids { + id, _ := strconv.Atoi(idStr) + existingMovies = append(existingMovies, models.MoviesScenes{MovieID: id}) + } + + return existingMovies, nil + } + + for _, idStr := range updateIDs.Ids { + id, _ := strconv.Atoi(idStr) + + // look for the id in the list + foundExisting := false + for idx, existingMovie := range existingMovies { + if existingMovie.MovieID == id { + if updateIDs.Mode == models.BulkUpdateIDModeRemove { + // remove from the list + existingMovies = append(existingMovies[:idx], existingMovies[idx+1:]...) + } + + foundExisting = true + break + } + } + + if !foundExisting && updateIDs.Mode != models.BulkUpdateIDModeRemove { + existingMovies = append(existingMovies, models.MoviesScenes{MovieID: id}) + } + } + + return existingMovies, err +} + func (r *mutationResolver) SceneDestroy(ctx context.Context, input models.SceneDestroyInput) (bool, error) { sceneID, err := strconv.Atoi(input.ID) if err != nil { diff --git a/pkg/api/resolver_mutation_studio.go b/pkg/api/resolver_mutation_studio.go index 108d952bc..bdca6059f 100644 --- a/pkg/api/resolver_mutation_studio.go +++ b/pkg/api/resolver_mutation_studio.go @@ -3,6 +3,7 @@ package api import ( "context" "database/sql" + "github.com/stashapp/stash/pkg/studio" "strconv" "time" @@ -64,19 +65,19 @@ func (r *mutationResolver) StudioCreate(ctx context.Context, input models.Studio } // Start the transaction and save the studio - var studio *models.Studio + var s *models.Studio if err := r.withTxn(ctx, func(repo models.Repository) error { qb := repo.Studio() var err error - studio, err = qb.Create(newStudio) + s, err = qb.Create(newStudio) if err != nil { return err } // update image table if len(imageData) > 0 { - if err := qb.UpdateImage(studio.ID, imageData); err != nil { + if err := qb.UpdateImage(s.ID, imageData); err != nil { return err } } @@ -84,7 +85,17 @@ func (r *mutationResolver) StudioCreate(ctx context.Context, input models.Studio // Save the stash_ids if input.StashIds != nil { stashIDJoins := models.StashIDsFromInput(input.StashIds) - if err := qb.UpdateStashIDs(studio.ID, stashIDJoins); err != nil { + if err := qb.UpdateStashIDs(s.ID, stashIDJoins); err != nil { + return err + } + } + + if len(input.Aliases) > 0 { + if err := studio.EnsureAliasesUnique(s.ID, input.Aliases, qb); err != nil { + return err + } + + if err := qb.UpdateAliases(s.ID, input.Aliases); err != nil { return err } } @@ -94,8 +105,8 @@ func (r *mutationResolver) StudioCreate(ctx context.Context, input models.Studio return nil, err } - r.hookExecutor.ExecutePostHooks(ctx, studio.ID, plugin.StudioCreatePost, input, nil) - return r.getStudio(ctx, studio.ID) + r.hookExecutor.ExecutePostHooks(ctx, s.ID, plugin.StudioCreatePost, input, nil) + return r.getStudio(ctx, s.ID) } func (r *mutationResolver) StudioUpdate(ctx context.Context, input models.StudioUpdateInput) (*models.Studio, error) { @@ -136,7 +147,7 @@ func (r *mutationResolver) StudioUpdate(ctx context.Context, input models.Studio updatedStudio.Rating = translator.nullInt64(input.Rating, "rating") // Start the transaction and save the studio - var studio *models.Studio + var s *models.Studio if err := r.withTxn(ctx, func(repo models.Repository) error { qb := repo.Studio() @@ -145,19 +156,19 @@ func (r *mutationResolver) StudioUpdate(ctx context.Context, input models.Studio } var err error - studio, err = qb.Update(updatedStudio) + s, err = qb.Update(updatedStudio) if err != nil { return err } // update image table if len(imageData) > 0 { - if err := qb.UpdateImage(studio.ID, imageData); err != nil { + if err := qb.UpdateImage(s.ID, imageData); err != nil { return err } } else if imageIncluded { // must be unsetting - if err := qb.DestroyImage(studio.ID); err != nil { + if err := qb.DestroyImage(s.ID); err != nil { return err } } @@ -170,13 +181,23 @@ func (r *mutationResolver) StudioUpdate(ctx context.Context, input models.Studio } } + if translator.hasField("aliases") { + if err := studio.EnsureAliasesUnique(studioID, input.Aliases, qb); err != nil { + return err + } + + if err := qb.UpdateAliases(studioID, input.Aliases); err != nil { + return err + } + } + return nil }); err != nil { return nil, err } - r.hookExecutor.ExecutePostHooks(ctx, studio.ID, plugin.StudioUpdatePost, input, translator.getFields()) - return r.getStudio(ctx, studio.ID) + r.hookExecutor.ExecutePostHooks(ctx, s.ID, plugin.StudioUpdatePost, input, translator.getFields()) + return r.getStudio(ctx, s.ID) } func (r *mutationResolver) StudioDestroy(ctx context.Context, input models.StudioDestroyInput) (bool, error) { diff --git a/pkg/api/resolver_mutation_tag.go b/pkg/api/resolver_mutation_tag.go index 94292a7ba..d1dc230e4 100644 --- a/pkg/api/resolver_mutation_tag.go +++ b/pkg/api/resolver_mutation_tag.go @@ -75,6 +75,28 @@ func (r *mutationResolver) TagCreate(ctx context.Context, input models.TagCreate } } + if input.ParentIds != nil && len(input.ParentIds) > 0 { + ids, err := utils.StringSliceToIntSlice(input.ParentIds) + if err != nil { + return err + } + + if err := qb.UpdateParentTags(t.ID, ids); err != nil { + return err + } + } + + if input.ChildIds != nil && len(input.ChildIds) > 0 { + ids, err := utils.StringSliceToIntSlice(input.ChildIds) + if err != nil { + return err + } + + if err := qb.UpdateChildTags(t.ID, ids); err != nil { + return err + } + } + return nil }); err != nil { return nil, err @@ -161,6 +183,41 @@ func (r *mutationResolver) TagUpdate(ctx context.Context, input models.TagUpdate } } + var parentIDs []int + var childIDs []int + + if translator.hasField("parent_ids") { + parentIDs, err = utils.StringSliceToIntSlice(input.ParentIds) + if err != nil { + return err + } + } + + if translator.hasField("child_ids") { + childIDs, err = utils.StringSliceToIntSlice(input.ChildIds) + if err != nil { + return err + } + } + + if parentIDs != nil || childIDs != nil { + if err := tag.EnsureUniqueHierarchy(tagID, parentIDs, childIDs, qb); err != nil { + return err + } + } + + if parentIDs != nil { + if err := qb.UpdateParentTags(tagID, parentIDs); err != nil { + return err + } + } + + if childIDs != nil { + if err := qb.UpdateChildTags(tagID, childIDs); err != nil { + return err + } + } + return nil }); err != nil { return nil, err @@ -242,10 +299,24 @@ func (r *mutationResolver) TagsMerge(ctx context.Context, input models.TagsMerge return fmt.Errorf("Tag with ID %d not found", destination) } + parents, children, err := tag.MergeHierarchy(destination, source, qb) + if err != nil { + return err + } + if err = qb.Merge(source, destination); err != nil { return err } + err = qb.UpdateParentTags(destination, parents) + if err != nil { + return err + } + err = qb.UpdateChildTags(destination, children) + if err != nil { + return err + } + return nil }); err != nil { return nil, err diff --git a/pkg/api/resolver_query_configuration.go b/pkg/api/resolver_query_configuration.go index 27aaac925..283148113 100644 --- a/pkg/api/resolver_query_configuration.go +++ b/pkg/api/resolver_query_configuration.go @@ -59,6 +59,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult { Stashes: config.GetStashPaths(), DatabasePath: config.GetDatabasePath(), GeneratedPath: config.GetGeneratedPath(), + MetadataPath: config.GetMetadataPath(), ConfigFilePath: config.GetConfigFilePath(), ScrapersPath: config.GetScrapersPath(), CachePath: config.GetCachePath(), @@ -73,10 +74,12 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult { PreviewPreset: config.GetPreviewPreset(), MaxTranscodeSize: &maxTranscodeSize, MaxStreamingTranscodeSize: &maxStreamingTranscodeSize, + WriteImageThumbnails: config.IsWriteImageThumbnails(), APIKey: config.GetAPIKey(), Username: config.GetUsername(), Password: config.GetPasswordHash(), MaxSessionAge: config.GetMaxSessionAge(), + TrustedProxies: config.GetTrustedProxies(), LogFile: &logFile, LogOut: config.GetLogOut(), LogLevel: config.GetLogLevel(), diff --git a/pkg/api/resolver_query_scraper.go b/pkg/api/resolver_query_scraper.go index 301870351..181363e24 100644 --- a/pkg/api/resolver_query_scraper.go +++ b/pkg/api/resolver_query_scraper.go @@ -2,7 +2,9 @@ package api import ( "context" + "errors" "fmt" + "strconv" "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/manager/config" @@ -29,8 +31,9 @@ func (r *queryResolver) ScrapeFreeonesPerformerList(ctx context.Context, query s var ret []string for _, v := range scrapedPerformers { - name := v.Name - ret = append(ret, *name) + if v.Name != nil { + ret = append(ret, *v.Name) + } } return ret, nil @@ -68,8 +71,21 @@ func (r *queryResolver) ScrapePerformerURL(ctx context.Context, url string) (*mo return manager.GetInstance().ScraperCache.ScrapePerformerURL(url) } +func (r *queryResolver) ScrapeSceneQuery(ctx context.Context, scraperID string, query string) ([]*models.ScrapedScene, error) { + if query == "" { + return nil, nil + } + + return manager.GetInstance().ScraperCache.ScrapeSceneQuery(scraperID, query) +} + func (r *queryResolver) ScrapeScene(ctx context.Context, scraperID string, scene models.SceneUpdateInput) (*models.ScrapedScene, error) { - return manager.GetInstance().ScraperCache.ScrapeScene(scraperID, scene) + id, err := strconv.Atoi(scene.ID) + if err != nil { + return nil, err + } + + return manager.GetInstance().ScraperCache.ScrapeScene(scraperID, id) } func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*models.ScrapedScene, error) { @@ -77,7 +93,12 @@ func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*models } func (r *queryResolver) ScrapeGallery(ctx context.Context, scraperID string, gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) { - return manager.GetInstance().ScraperCache.ScrapeGallery(scraperID, gallery) + id, err := strconv.Atoi(gallery.ID) + if err != nil { + return nil, err + } + + return manager.GetInstance().ScraperCache.ScrapeGallery(scraperID, id) } func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*models.ScrapedGallery, error) { @@ -98,7 +119,7 @@ func (r *queryResolver) QueryStashBoxScene(ctx context.Context, input models.Sta client := stashbox.NewClient(*boxes[input.StashBoxIndex], r.txnManager) if len(input.SceneIds) > 0 { - return client.FindStashBoxScenesByFingerprints(input.SceneIds) + return client.FindStashBoxScenesByFingerprintsFlat(input.SceneIds) } if input.Q != nil { @@ -127,3 +148,177 @@ func (r *queryResolver) QueryStashBoxPerformer(ctx context.Context, input models return nil, nil } + +func (r *queryResolver) getStashBoxClient(index int) (*stashbox.Client, error) { + boxes := config.GetInstance().GetStashBoxes() + + if index < 0 || index >= len(boxes) { + return nil, fmt.Errorf("invalid stash_box_index %d", index) + } + + return stashbox.NewClient(*boxes[index], r.txnManager), nil +} + +func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSingleSceneInput) ([]*models.ScrapedScene, error) { + if source.ScraperID != nil { + var singleScene *models.ScrapedScene + var err error + + if input.SceneID != nil { + var sceneID int + sceneID, err = strconv.Atoi(*input.SceneID) + if err != nil { + return nil, err + } + singleScene, err = manager.GetInstance().ScraperCache.ScrapeScene(*source.ScraperID, sceneID) + } else if input.SceneInput != nil { + singleScene, err = manager.GetInstance().ScraperCache.ScrapeSceneFragment(*source.ScraperID, *input.SceneInput) + } else if input.Query != nil { + return manager.GetInstance().ScraperCache.ScrapeSceneQuery(*source.ScraperID, *input.Query) + } else { + err = errors.New("scene_id, scene_input or query must be set") + } + + if err != nil { + return nil, err + } + + if singleScene != nil { + return []*models.ScrapedScene{singleScene}, nil + } + + return nil, nil + } else if source.StashBoxIndex != nil { + client, err := r.getStashBoxClient(*source.StashBoxIndex) + if err != nil { + return nil, err + } + + if input.SceneID != nil { + return client.FindStashBoxScenesByFingerprintsFlat([]string{*input.SceneID}) + } else if input.Query != nil { + return client.QueryStashBoxScene(*input.Query) + } + + return nil, errors.New("scene_id or query must be set") + } + + return nil, errors.New("scraper_id or stash_box_index must be set") +} + +func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeMultiScenesInput) ([][]*models.ScrapedScene, error) { + if source.ScraperID != nil { + return nil, errors.New("not implemented") + } else if source.StashBoxIndex != nil { + client, err := r.getStashBoxClient(*source.StashBoxIndex) + if err != nil { + return nil, err + } + + return client.FindStashBoxScenesByFingerprints(input.SceneIds) + } + + return nil, errors.New("scraper_id or stash_box_index must be set") +} + +func (r *queryResolver) ScrapeSinglePerformer(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSinglePerformerInput) ([]*models.ScrapedPerformer, error) { + if source.ScraperID != nil { + if input.PerformerInput != nil { + singlePerformer, err := manager.GetInstance().ScraperCache.ScrapePerformer(*source.ScraperID, *input.PerformerInput) + if err != nil { + return nil, err + } + + if singlePerformer != nil { + return []*models.ScrapedPerformer{singlePerformer}, nil + } + + return nil, nil + } + + if input.Query != nil { + return manager.GetInstance().ScraperCache.ScrapePerformerList(*source.ScraperID, *input.Query) + } + + return nil, errors.New("not implemented") + } else if source.StashBoxIndex != nil { + client, err := r.getStashBoxClient(*source.StashBoxIndex) + if err != nil { + return nil, err + } + + var ret []*models.StashBoxPerformerQueryResult + if input.PerformerID != nil { + ret, err = client.FindStashBoxPerformersByNames([]string{*input.PerformerID}) + } else if input.Query != nil { + ret, err = client.QueryStashBoxPerformer(*input.Query) + } else { + return nil, errors.New("not implemented") + } + + if err != nil { + return nil, err + } + + if len(ret) > 0 { + return ret[0].Results, nil + } + + return nil, nil + } + + return nil, errors.New("scraper_id or stash_box_index must be set") +} + +func (r *queryResolver) ScrapeMultiPerformers(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeMultiPerformersInput) ([][]*models.ScrapedPerformer, error) { + if source.ScraperID != nil { + return nil, errors.New("not implemented") + } else if source.StashBoxIndex != nil { + client, err := r.getStashBoxClient(*source.StashBoxIndex) + if err != nil { + return nil, err + } + + return client.FindStashBoxPerformersByPerformerNames(input.PerformerIds) + } + + return nil, errors.New("scraper_id or stash_box_index must be set") +} + +func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSingleGalleryInput) ([]*models.ScrapedGallery, error) { + if source.ScraperID != nil { + var singleGallery *models.ScrapedGallery + var err error + + if input.GalleryID != nil { + var galleryID int + galleryID, err = strconv.Atoi(*input.GalleryID) + if err != nil { + return nil, err + } + singleGallery, err = manager.GetInstance().ScraperCache.ScrapeGallery(*source.ScraperID, galleryID) + } else if input.GalleryInput != nil { + singleGallery, err = manager.GetInstance().ScraperCache.ScrapeGalleryFragment(*source.ScraperID, *input.GalleryInput) + } else { + return nil, errors.New("not implemented") + } + + if err != nil { + return nil, err + } + + if singleGallery != nil { + return []*models.ScrapedGallery{singleGallery}, nil + } + + return nil, nil + } else if source.StashBoxIndex != nil { + return nil, errors.New("not supported") + } + + return nil, errors.New("scraper_id must be set") +} + +func (r *queryResolver) ScrapeSingleMovie(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSingleMovieInput) ([]*models.ScrapedMovie, error) { + return nil, errors.New("not supported") +} diff --git a/pkg/api/resolver_subscription_job.go b/pkg/api/resolver_subscription_job.go index c7b3176c0..2ee28df96 100644 --- a/pkg/api/resolver_subscription_job.go +++ b/pkg/api/resolver_subscription_job.go @@ -2,32 +2,12 @@ package api import ( "context" - "time" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" ) -type throttledUpdate struct { - id int - pendingUpdate *job.Job - lastUpdate time.Time - broadcastTimer *time.Timer - killTimer *time.Timer -} - -func (tu *throttledUpdate) broadcast(output chan *models.JobStatusUpdate) { - tu.lastUpdate = time.Now() - output <- &models.JobStatusUpdate{ - Type: models.JobStatusUpdateTypeUpdate, - Job: jobToJobModel(*tu.pendingUpdate), - } - - tu.broadcastTimer = nil - tu.pendingUpdate = nil -} - func makeJobStatusUpdate(t models.JobStatusUpdateType, j job.Job) *models.JobStatusUpdate { return &models.JobStatusUpdate{ Type: t, diff --git a/pkg/api/routes_image.go b/pkg/api/routes_image.go index 3e64887a8..9cdaa6653 100644 --- a/pkg/api/routes_image.go +++ b/pkg/api/routes_image.go @@ -7,6 +7,7 @@ import ( "github.com/go-chi/chi" "github.com/stashapp/stash/pkg/image" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" @@ -32,15 +33,35 @@ func (rs imageRoutes) Routes() chi.Router { // region Handlers func (rs imageRoutes) Thumbnail(w http.ResponseWriter, r *http.Request) { - image := r.Context().Value(imageKey).(*models.Image) - filepath := manager.GetInstance().Paths.Generated.GetThumbnailPath(image.Checksum, models.DefaultGthumbWidth) + img := r.Context().Value(imageKey).(*models.Image) + filepath := manager.GetInstance().Paths.Generated.GetThumbnailPath(img.Checksum, models.DefaultGthumbWidth) - // if the thumbnail doesn't exist, fall back to the original file + w.Header().Add("Cache-Control", "max-age=604800000") + + // if the thumbnail doesn't exist, encode on the fly exists, _ := utils.FileExists(filepath) if exists { http.ServeFile(w, r, filepath) } else { - rs.Image(w, r) + encoder := image.NewThumbnailEncoder(manager.GetInstance().FFMPEGPath) + data, err := encoder.GetThumbnail(img, models.DefaultGthumbWidth) + if err != nil { + logger.Errorf("error generating thumbnail for image: %s", err.Error()) + + // backwards compatibility - fallback to original image instead + rs.Image(w, r) + return + } + + // write the generated thumbnail to disk if enabled + if manager.GetInstance().Config.IsWriteImageThumbnails() { + if err := utils.WriteFile(filepath, data); err != nil { + logger.Errorf("error writing thumbnail for image %s: %s", img.Path, err) + } + } + if n, err := w.Write(data); err != nil { + logger.Errorf("error writing thumbnail response. Wrote %v bytes: %v", n, err) + } } } @@ -59,7 +80,7 @@ func ImageCtx(next http.Handler) http.Handler { imageID, _ := strconv.Atoi(imageIdentifierQueryParam) var image *models.Image - manager.GetInstance().TxnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { + readTxnErr := manager.GetInstance().TxnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { qb := repo.Image() if imageID == 0 { image, _ = qb.FindByChecksum(imageIdentifierQueryParam) @@ -69,6 +90,9 @@ func ImageCtx(next http.Handler) http.Handler { return nil }) + if readTxnErr != nil { + logger.Warnf("read transaction failure while trying to read image by id: %v", readTxnErr) + } if image == nil { http.Error(w, http.StatusText(404), 404) diff --git a/pkg/api/routes_movie.go b/pkg/api/routes_movie.go index 988d4ddf9..4018d39ed 100644 --- a/pkg/api/routes_movie.go +++ b/pkg/api/routes_movie.go @@ -6,6 +6,7 @@ import ( "strconv" "github.com/go-chi/chi" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" @@ -32,17 +33,22 @@ func (rs movieRoutes) FrontImage(w http.ResponseWriter, r *http.Request) { defaultParam := r.URL.Query().Get("default") var image []byte if defaultParam != "true" { - rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { + err := rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { image, _ = repo.Movie().GetFrontImage(movie.ID) return nil }) + if err != nil { + logger.Warnf("read transaction error while getting front image: %v", err) + } } if len(image) == 0 { _, image, _ = utils.ProcessBase64Image(models.DefaultMovieImage) } - utils.ServeImage(image, w, r) + if err := utils.ServeImage(image, w, r); err != nil { + logger.Warnf("error serving front image: %v", err) + } } func (rs movieRoutes) BackImage(w http.ResponseWriter, r *http.Request) { @@ -50,17 +56,22 @@ func (rs movieRoutes) BackImage(w http.ResponseWriter, r *http.Request) { defaultParam := r.URL.Query().Get("default") var image []byte if defaultParam != "true" { - rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { + err := rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { image, _ = repo.Movie().GetBackImage(movie.ID) return nil }) + if err != nil { + logger.Warnf("read transaction error on fetch back image: %v", err) + } } if len(image) == 0 { _, image, _ = utils.ProcessBase64Image(models.DefaultMovieImage) } - utils.ServeImage(image, w, r) + if err := utils.ServeImage(image, w, r); err != nil { + logger.Warnf("error while serving image: %v", err) + } } func MovieCtx(next http.Handler) http.Handler { diff --git a/pkg/api/routes_performer.go b/pkg/api/routes_performer.go index 1940e0dfe..101fb2932 100644 --- a/pkg/api/routes_performer.go +++ b/pkg/api/routes_performer.go @@ -6,6 +6,7 @@ import ( "strconv" "github.com/go-chi/chi" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" @@ -33,17 +34,22 @@ func (rs performerRoutes) Image(w http.ResponseWriter, r *http.Request) { var image []byte if defaultParam != "true" { - rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { + readTxnErr := rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { image, _ = repo.Performer().GetImage(performer.ID) return nil }) + if readTxnErr != nil { + logger.Warnf("couldn't execute getting a performer image from read transaction: %v", readTxnErr) + } } if len(image) == 0 || defaultParam == "true" { image, _ = getRandomPerformerImageUsingName(performer.Name.String, performer.Gender.String, config.GetInstance().GetCustomPerformerImageLocation()) } - utils.ServeImage(image, w, r) + if err := utils.ServeImage(image, w, r); err != nil { + logger.Warnf("error serving image: %v", err) + } } func PerformerCtx(next http.Handler) http.Handler { diff --git a/pkg/api/routes_scene.go b/pkg/api/routes_scene.go index d7bb1d888..a029b79a9 100644 --- a/pkg/api/routes_scene.go +++ b/pkg/api/routes_scene.go @@ -16,8 +16,7 @@ import ( ) type sceneRoutes struct { - txnManager models.TransactionManager - sceneServer manager.SceneServer + txnManager models.TransactionManager } func (rs sceneRoutes) Routes() chi.Router { @@ -42,6 +41,7 @@ func (rs sceneRoutes) Routes() chi.Router { r.Get("/scene_marker/{sceneMarkerId}/stream", rs.SceneMarkerStream) r.Get("/scene_marker/{sceneMarkerId}/preview", rs.SceneMarkerPreview) + r.Get("/scene_marker/{sceneMarkerId}/screenshot", rs.SceneMarkerScreenshot) }) r.With(SceneCtx).Get("/{sceneId}_thumbs.vtt", rs.VttThumbs) r.With(SceneCtx).Get("/{sceneId}_sprite.jpg", rs.VttSprite) @@ -59,7 +59,7 @@ func getSceneFileContainer(scene *models.Scene) ffmpeg.Container { // shouldn't happen, fallback to ffprobe tmpVideoFile, err := ffmpeg.NewVideoFile(manager.GetInstance().FFProbePath, scene.Path, false) if err != nil { - logger.Errorf("[transcode] error reading video file: %s", err.Error()) + logger.Errorf("[transcode] error reading video file: %v", err) return ffmpeg.Container("") } @@ -85,7 +85,9 @@ func (rs sceneRoutes) StreamMKV(w http.ResponseWriter, r *http.Request) { container := getSceneFileContainer(scene) if container != ffmpeg.Matroska { w.WriteHeader(http.StatusBadRequest) - w.Write([]byte("not an mkv file")) + if _, err := w.Write([]byte("not an mkv file")); err != nil { + logger.Warnf("[stream] error writing to stream: %v", err) + } return } @@ -105,7 +107,7 @@ func (rs sceneRoutes) StreamHLS(w http.ResponseWriter, r *http.Request) { videoFile, err := ffmpeg.NewVideoFile(manager.GetInstance().FFProbePath, scene.Path, false) if err != nil { - logger.Errorf("[stream] error reading video file: %s", err.Error()) + logger.Errorf("[stream] error reading video file: %v", err) return } @@ -126,7 +128,9 @@ func (rs sceneRoutes) StreamHLS(w http.ResponseWriter, r *http.Request) { rangeStr := requestByteRange.ToHeaderValue(int64(str.Len())) w.Header().Set("Content-Range", rangeStr) - w.Write(ret) + if n, err := w.Write(ret); err != nil { + logger.Warnf("[stream] error writing stream (wrote %v bytes): %v", n, err) + } } func (rs sceneRoutes) StreamTS(w http.ResponseWriter, r *http.Request) { @@ -141,12 +145,15 @@ func (rs sceneRoutes) streamTranscode(w http.ResponseWriter, r *http.Request, vi videoFile, err := ffmpeg.NewVideoFile(manager.GetInstance().FFProbePath, scene.Path, false) if err != nil { - logger.Errorf("[stream] error reading video file: %s", err.Error()) + logger.Errorf("[stream] error reading video file: %v", err) return } // start stream based on query param, if provided - r.ParseForm() + if err = r.ParseForm(); err != nil { + logger.Warnf("[stream] error parsing query form: %v", err) + } + startTime := r.Form.Get("start") requestedSize := r.Form.Get("resolution") @@ -168,9 +175,11 @@ func (rs sceneRoutes) streamTranscode(w http.ResponseWriter, r *http.Request, vi stream, err = encoder.GetTranscodeStream(options) if err != nil { - logger.Errorf("[stream] error transcoding video file: %s", err.Error()) + logger.Errorf("[stream] error transcoding video file: %v", err) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(err.Error())) + if _, err := w.Write([]byte(err.Error())); err != nil { + logger.Warnf("[stream] error writing response: %v", err) + } return } @@ -289,6 +298,12 @@ func (rs sceneRoutes) SceneMarkerStream(w http.ResponseWriter, r *http.Request) http.Error(w, http.StatusText(500), 500) return } + + if sceneMarker == nil { + http.Error(w, http.StatusText(404), 404) + return + } + filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), int(sceneMarker.Seconds)) http.ServeFile(w, r, filepath) } @@ -306,6 +321,12 @@ func (rs sceneRoutes) SceneMarkerPreview(w http.ResponseWriter, r *http.Request) http.Error(w, http.StatusText(500), 500) return } + + if sceneMarker == nil { + http.Error(w, http.StatusText(404), 404) + return + } + filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPreviewImagePath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), int(sceneMarker.Seconds)) // If the image doesn't exist, send the placeholder @@ -320,6 +341,39 @@ func (rs sceneRoutes) SceneMarkerPreview(w http.ResponseWriter, r *http.Request) http.ServeFile(w, r, filepath) } +func (rs sceneRoutes) SceneMarkerScreenshot(w http.ResponseWriter, r *http.Request) { + scene := r.Context().Value(sceneKey).(*models.Scene) + sceneMarkerID, _ := strconv.Atoi(chi.URLParam(r, "sceneMarkerId")) + var sceneMarker *models.SceneMarker + if err := rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { + var err error + sceneMarker, err = repo.SceneMarker().Find(sceneMarkerID) + return err + }); err != nil { + logger.Warnf("Error when getting scene marker for stream: %s", err.Error()) + http.Error(w, http.StatusText(500), 500) + return + } + + if sceneMarker == nil { + http.Error(w, http.StatusText(404), 404) + return + } + + filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamScreenshotPath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), int(sceneMarker.Seconds)) + + // If the image doesn't exist, send the placeholder + exists, _ := utils.FileExists(filepath) + if !exists { + w.Header().Set("Content-Type", "image/png") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(utils.PendingGenerateResource) + return + } + + http.ServeFile(w, r, filepath) +} + // endregion func SceneCtx(next http.Handler) http.Handler { @@ -328,7 +382,7 @@ func SceneCtx(next http.Handler) http.Handler { sceneID, _ := strconv.Atoi(sceneIdentifierQueryParam) var scene *models.Scene - manager.GetInstance().TxnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { + readTxnErr := manager.GetInstance().TxnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { qb := repo.Scene() if sceneID == 0 { // determine checksum/os by the length of the query param @@ -343,6 +397,9 @@ func SceneCtx(next http.Handler) http.Handler { return nil }) + if readTxnErr != nil { + logger.Warnf("error executing SceneCtx transaction: %v", readTxnErr) + } if scene == nil { http.Error(w, http.StatusText(404), 404) diff --git a/pkg/api/routes_studio.go b/pkg/api/routes_studio.go index 840e060ad..67cb862dc 100644 --- a/pkg/api/routes_studio.go +++ b/pkg/api/routes_studio.go @@ -6,6 +6,7 @@ import ( "strconv" "github.com/go-chi/chi" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" @@ -32,17 +33,22 @@ func (rs studioRoutes) Image(w http.ResponseWriter, r *http.Request) { var image []byte if defaultParam != "true" { - rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { + err := rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { image, _ = repo.Studio().GetImage(studio.ID) return nil }) + if err != nil { + logger.Warnf("read transaction error while fetching studio image: %v", err) + } } if len(image) == 0 { _, image, _ = utils.ProcessBase64Image(models.DefaultStudioImage) } - utils.ServeImage(image, w, r) + if err := utils.ServeImage(image, w, r); err != nil { + logger.Warnf("error serving studio image: %v", err) + } } func StudioCtx(next http.Handler) http.Handler { diff --git a/pkg/api/routes_tag.go b/pkg/api/routes_tag.go index 42d64271c..cebb628cc 100644 --- a/pkg/api/routes_tag.go +++ b/pkg/api/routes_tag.go @@ -6,6 +6,7 @@ import ( "strconv" "github.com/go-chi/chi" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" @@ -32,17 +33,22 @@ func (rs tagRoutes) Image(w http.ResponseWriter, r *http.Request) { var image []byte if defaultParam != "true" { - rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { + err := rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { image, _ = repo.Tag().GetImage(tag.ID) return nil }) + if err != nil { + logger.Warnf("read transaction error while getting tag image: %v", err) + } } if len(image) == 0 { image = models.DefaultTagImage } - utils.ServeImage(image, w, r) + if err := utils.ServeImage(image, w, r); err != nil { + logger.Warnf("error serving tag image: %v", err) + } } func TagCtx(next http.Handler) http.Handler { diff --git a/pkg/api/server.go b/pkg/api/server.go index 823dae457..58310b4e2 100644 --- a/pkg/api/server.go +++ b/pkg/api/server.go @@ -3,11 +3,12 @@ package api import ( "context" "crypto/tls" + "embed" "errors" "fmt" - "io/ioutil" + "io/fs" "net/http" - "net/url" + "os" "path" "runtime/debug" "strconv" @@ -21,14 +22,12 @@ import ( gqlPlayground "github.com/99designs/gqlgen/graphql/playground" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" - "github.com/gobuffalo/packr/v2" "github.com/gorilla/websocket" "github.com/rs/cors" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/session" "github.com/stashapp/stash/pkg/utils" ) @@ -36,85 +35,7 @@ var version string var buildstamp string var githash string -var uiBox *packr.Box - -//var legacyUiBox *packr.Box -var loginUIBox *packr.Box - -func allowUnauthenticated(r *http.Request) bool { - return strings.HasPrefix(r.URL.Path, "/login") || r.URL.Path == "/css" -} - -func authenticateHandler() func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - userID, err := manager.GetInstance().SessionStore.Authenticate(w, r) - if err != nil { - if err != session.ErrUnauthorized { - w.WriteHeader(http.StatusInternalServerError) - _, err = w.Write([]byte(err.Error())) - if err != nil { - logger.Error(err) - } - return - } - - // unauthorized error - w.Header().Add("WWW-Authenticate", `FormBased`) - w.WriteHeader(http.StatusUnauthorized) - return - } - - c := config.GetInstance() - ctx := r.Context() - - // handle redirect if no user and user is required - if userID == "" && c.HasCredentials() && !allowUnauthenticated(r) { - // if we don't have a userID, then redirect - // if graphql was requested, we just return a forbidden error - if r.URL.Path == "/graphql" { - w.Header().Add("WWW-Authenticate", `FormBased`) - w.WriteHeader(http.StatusUnauthorized) - return - } - - // otherwise redirect to the login page - u := url.URL{ - Path: "/login", - } - q := u.Query() - q.Set(returnURLParam, r.URL.Path) - u.RawQuery = q.Encode() - http.Redirect(w, r, u.String(), http.StatusFound) - return - } - - ctx = session.SetCurrentUserID(ctx, userID) - - r = r.WithContext(ctx) - - next.ServeHTTP(w, r) - }) - } -} - -func visitedPluginHandler() func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // get the visited plugins and set them in the context - - next.ServeHTTP(w, r) - }) - } -} - -const loginEndPoint = "/login" - -func Start() { - uiBox = packr.New("UI Box", "../../ui/v2.5/build") - //legacyUiBox = packr.New("UI Box", "../../ui/v1/dist/stash-frontend") - loginUIBox = packr.New("Login UI Box", "../../ui/login") - +func Start(uiBox embed.FS, loginUIBox embed.FS) { initialiseImages() r := chi.NewRouter() @@ -182,10 +103,10 @@ func Start() { r.HandleFunc("/playground", gqlPlayground.Handler("GraphQL playground", "/graphql")) // session handlers - r.Post(loginEndPoint, handleLogin) - r.Get("/logout", handleLogout) + r.Post(loginEndPoint, handleLogin(loginUIBox)) + r.Get("/logout", handleLogout(loginUIBox)) - r.Get(loginEndPoint, getLoginHandler) + r.Get(loginEndPoint, getLoginHandler(loginUIBox)) r.Mount("/performer", performerRoutes{ txnManager: txnManager, @@ -226,11 +147,18 @@ func Start() { r.HandleFunc("/login*", func(w http.ResponseWriter, r *http.Request) { ext := path.Ext(r.URL.Path) if ext == ".html" || ext == "" { - data, _ := loginUIBox.Find("login.html") - _, _ = w.Write(data) + prefix := getProxyPrefix(r.Header) + + data := getLoginPage(loginUIBox) + baseURLIndex := strings.Replace(string(data), "%BASE_URL%", prefix+"/", 2) + _, _ = w.Write([]byte(baseURLIndex)) } else { r.URL.Path = strings.Replace(r.URL.Path, loginEndPoint, "", 1) - http.FileServer(loginUIBox).ServeHTTP(w, r) + loginRoot, err := fs.Sub(loginUIBox, loginRootDir) + if err != nil { + panic(err) + } + http.FileServer(http.FS(loginRoot)).ServeHTTP(w, r) } }) @@ -255,6 +183,8 @@ func Start() { // Serve the web app r.HandleFunc("/*", func(w http.ResponseWriter, r *http.Request) { + const uiRootDir = "ui/v2.5/build" + ext := path.Ext(r.URL.Path) if customUILocation != "" { @@ -267,14 +197,25 @@ func Start() { } if ext == ".html" || ext == "" { - data, _ := uiBox.Find("index.html") - _, _ = w.Write(data) + data, err := uiBox.ReadFile(uiRootDir + "/index.html") + if err != nil { + panic(err) + } + + prefix := getProxyPrefix(r.Header) + baseURLIndex := strings.Replace(string(data), "%BASE_URL%", prefix+"/", 2) + baseURLIndex = strings.Replace(baseURLIndex, "base href=\"/\"", fmt.Sprintf("base href=\"%s\"", prefix+"/"), 2) + _, _ = w.Write([]byte(baseURLIndex)) } else { isStatic, _ := path.Match("/static/*/*", r.URL.Path) if isStatic { w.Header().Add("Cache-Control", "max-age=604800000") } - http.FileServer(uiBox).ServeHTTP(w, r) + uiRoot, err := fs.Sub(uiBox, uiRootDir) + if err != nil { + panic(err) + } + http.FileServer(http.FS(uiRoot)).ServeHTTP(w, r) } }) @@ -342,12 +283,12 @@ func makeTLSConfig(c *config.Instance) (*tls.Config, error) { return nil, errors.New("SSL key file must be present if certificate file is present") } - cert, err := ioutil.ReadFile(certFile) + cert, err := os.ReadFile(certFile) if err != nil { return nil, fmt.Errorf("error reading SSL certificate file %s: %s", certFile, err.Error()) } - key, err := ioutil.ReadFile(keyFile) + key, err := os.ReadFile(keyFile) if err != nil { return nil, fmt.Errorf("error reading SSL key file %s: %s", keyFile, err.Error()) } @@ -382,11 +323,19 @@ func BaseURLMiddleware(next http.Handler) http.Handler { } else { scheme = "http" } - baseURL := scheme + "://" + r.Host + prefix := getProxyPrefix(r.Header) + + port := "" + forwardedPort := r.Header.Get("X-Forwarded-Port") + if forwardedPort != "" && forwardedPort != "80" && forwardedPort != "8080" { + port = ":" + forwardedPort + } + + baseURL := scheme + "://" + r.Host + port + prefix externalHost := config.GetInstance().GetExternalHost() if externalHost != "" { - baseURL = externalHost + baseURL = externalHost + prefix } r = r.WithContext(context.WithValue(ctx, BaseURLCtxKey, baseURL)) @@ -395,3 +344,12 @@ func BaseURLMiddleware(next http.Handler) http.Handler { } return http.HandlerFunc(fn) } + +func getProxyPrefix(headers http.Header) string { + prefix := "" + if headers.Get("X-Forwarded-Prefix") != "" { + prefix = strings.TrimRight(headers.Get("X-Forwarded-Prefix"), "/") + } + + return prefix +} diff --git a/pkg/api/session.go b/pkg/api/session.go index 739df916f..71fa15136 100644 --- a/pkg/api/session.go +++ b/pkg/api/session.go @@ -1,6 +1,7 @@ package api import ( + "embed" "fmt" "html/template" "net/http" @@ -10,20 +11,24 @@ import ( "github.com/stashapp/stash/pkg/session" ) -const cookieName = "session" -const usernameFormKey = "username" -const passwordFormKey = "password" -const userIDKey = "userID" - +const loginRootDir = "ui/login" const returnURLParam = "returnURL" +func getLoginPage(loginUIBox embed.FS) []byte { + data, err := loginUIBox.ReadFile(loginRootDir + "/login.html") + if err != nil { + panic(err) + } + return data +} + type loginTemplateData struct { URL string Error string } -func redirectToLogin(w http.ResponseWriter, returnURL string, loginError string) { - data, _ := loginUIBox.Find("login.html") +func redirectToLogin(loginUIBox embed.FS, w http.ResponseWriter, returnURL string, loginError string) { + data := getLoginPage(loginUIBox) templ, err := template.New("Login").Parse(string(data)) if err != nil { http.Error(w, fmt.Sprintf("error: %s", err), http.StatusInternalServerError) @@ -36,42 +41,48 @@ func redirectToLogin(w http.ResponseWriter, returnURL string, loginError string) } } -func getLoginHandler(w http.ResponseWriter, r *http.Request) { - if !config.GetInstance().HasCredentials() { - http.Redirect(w, r, "/", http.StatusFound) - return - } +func getLoginHandler(loginUIBox embed.FS) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !config.GetInstance().HasCredentials() { + http.Redirect(w, r, "/", http.StatusFound) + return + } - redirectToLogin(w, r.URL.Query().Get(returnURLParam), "") + redirectToLogin(loginUIBox, w, r.URL.Query().Get(returnURLParam), "") + } } -func handleLogin(w http.ResponseWriter, r *http.Request) { - url := r.FormValue(returnURLParam) - if url == "" { - url = "/" - } +func handleLogin(loginUIBox embed.FS) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + url := r.FormValue(returnURLParam) + if url == "" { + url = "/" + } - err := manager.GetInstance().SessionStore.Login(w, r) - if err == session.ErrInvalidCredentials { - // redirect back to the login page with an error - redirectToLogin(w, url, "Username or password is invalid") - return - } + err := manager.GetInstance().SessionStore.Login(w, r) + if err == session.ErrInvalidCredentials { + // redirect back to the login page with an error + redirectToLogin(loginUIBox, w, url, "Username or password is invalid") + return + } - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } - http.Redirect(w, r, url, http.StatusFound) + http.Redirect(w, r, url, http.StatusFound) + } } -func handleLogout(w http.ResponseWriter, r *http.Request) { - if err := manager.GetInstance().SessionStore.Logout(w, r); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } +func handleLogout(loginUIBox embed.FS) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if err := manager.GetInstance().SessionStore.Logout(w, r); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } - // redirect to the login page if credentials are required - getLoginHandler(w, r) + // redirect to the login page if credentials are required + getLoginHandler(loginUIBox)(w, r) + } } diff --git a/pkg/api/urlbuilders/scene.go b/pkg/api/urlbuilders/scene.go index e9766da9e..8e45feb6e 100644 --- a/pkg/api/urlbuilders/scene.go +++ b/pkg/api/urlbuilders/scene.go @@ -59,6 +59,10 @@ func (b SceneURLBuilder) GetSceneMarkerStreamPreviewURL(sceneMarkerID int) strin return b.BaseURL + "/scene/" + b.SceneID + "/scene_marker/" + strconv.Itoa(sceneMarkerID) + "/preview" } +func (b SceneURLBuilder) GetSceneMarkerStreamScreenshotURL(sceneMarkerID int) string { + return b.BaseURL + "/scene/" + b.SceneID + "/scene_marker/" + strconv.Itoa(sceneMarkerID) + "/screenshot" +} + func (b SceneURLBuilder) GetFunscriptURL() string { return b.BaseURL + "/scene/" + b.SceneID + "/funscript" } diff --git a/pkg/autotag/gallery_test.go b/pkg/autotag/gallery_test.go index ff47f20c1..f2f498519 100644 --- a/pkg/autotag/gallery_test.go +++ b/pkg/autotag/gallery_test.go @@ -12,6 +12,8 @@ import ( const galleryExt = "zip" func TestGalleryPerformers(t *testing.T) { + t.Parallel() + const galleryID = 1 const performerName = "performer name" const performerID = 2 @@ -55,6 +57,8 @@ func TestGalleryPerformers(t *testing.T) { } func TestGalleryStudios(t *testing.T) { + t.Parallel() + const galleryID = 1 const studioName = "studio name" const studioID = 2 @@ -74,12 +78,7 @@ func TestGalleryStudios(t *testing.T) { assert := assert.New(t) - for _, test := range testTables { - mockStudioReader := &mocks.StudioReaderWriter{} - mockGalleryReader := &mocks.GalleryReaderWriter{} - - mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() - + doTest := func(mockStudioReader *mocks.StudioReaderWriter, mockGalleryReader *mocks.GalleryReaderWriter, test pathTestTable) { if test.Matches { mockGalleryReader.On("Find", galleryID).Return(&models.Gallery{}, nil).Once() expectedStudioID := models.NullInt64(studioID) @@ -99,9 +98,38 @@ func TestGalleryStudios(t *testing.T) { mockStudioReader.AssertExpectations(t) mockGalleryReader.AssertExpectations(t) } + + for _, test := range testTables { + mockStudioReader := &mocks.StudioReaderWriter{} + mockGalleryReader := &mocks.GalleryReaderWriter{} + + mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() + mockStudioReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() + + doTest(mockStudioReader, mockGalleryReader, test) + } + + // test against aliases + const unmatchedName = "unmatched" + studio.Name.String = unmatchedName + + for _, test := range testTables { + mockStudioReader := &mocks.StudioReaderWriter{} + mockGalleryReader := &mocks.GalleryReaderWriter{} + + mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() + mockStudioReader.On("GetAliases", studioID).Return([]string{ + studioName, + }, nil).Once() + mockStudioReader.On("GetAliases", reversedStudioID).Return([]string{}, nil).Once() + + doTest(mockStudioReader, mockGalleryReader, test) + } } func TestGalleryTags(t *testing.T) { + t.Parallel() + const galleryID = 1 const tagName = "tag name" const tagID = 2 @@ -121,12 +149,7 @@ func TestGalleryTags(t *testing.T) { assert := assert.New(t) - for _, test := range testTables { - mockTagReader := &mocks.TagReaderWriter{} - mockGalleryReader := &mocks.GalleryReaderWriter{} - - mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() - + doTest := func(mockTagReader *mocks.TagReaderWriter, mockGalleryReader *mocks.GalleryReaderWriter, test pathTestTable) { if test.Matches { mockGalleryReader.On("GetTagIDs", galleryID).Return(nil, nil).Once() mockGalleryReader.On("UpdateTags", galleryID, []int{tagID}).Return(nil).Once() @@ -142,4 +165,30 @@ func TestGalleryTags(t *testing.T) { mockTagReader.AssertExpectations(t) mockGalleryReader.AssertExpectations(t) } + + for _, test := range testTables { + mockTagReader := &mocks.TagReaderWriter{} + mockGalleryReader := &mocks.GalleryReaderWriter{} + + mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() + mockTagReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() + + doTest(mockTagReader, mockGalleryReader, test) + } + + const unmatchedName = "unmatched" + tag.Name = unmatchedName + + for _, test := range testTables { + mockTagReader := &mocks.TagReaderWriter{} + mockGalleryReader := &mocks.GalleryReaderWriter{} + + mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() + mockTagReader.On("GetAliases", tagID).Return([]string{ + tagName, + }, nil).Once() + mockTagReader.On("GetAliases", reversedTagID).Return([]string{}, nil).Once() + + doTest(mockTagReader, mockGalleryReader, test) + } } diff --git a/pkg/autotag/image_test.go b/pkg/autotag/image_test.go index 8dba6b6e2..68b33ab74 100644 --- a/pkg/autotag/image_test.go +++ b/pkg/autotag/image_test.go @@ -12,6 +12,8 @@ import ( const imageExt = "jpg" func TestImagePerformers(t *testing.T) { + t.Parallel() + const imageID = 1 const performerName = "performer name" const performerID = 2 @@ -55,6 +57,8 @@ func TestImagePerformers(t *testing.T) { } func TestImageStudios(t *testing.T) { + t.Parallel() + const imageID = 1 const studioName = "studio name" const studioID = 2 @@ -74,12 +78,7 @@ func TestImageStudios(t *testing.T) { assert := assert.New(t) - for _, test := range testTables { - mockStudioReader := &mocks.StudioReaderWriter{} - mockImageReader := &mocks.ImageReaderWriter{} - - mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() - + doTest := func(mockStudioReader *mocks.StudioReaderWriter, mockImageReader *mocks.ImageReaderWriter, test pathTestTable) { if test.Matches { mockImageReader.On("Find", imageID).Return(&models.Image{}, nil).Once() expectedStudioID := models.NullInt64(studioID) @@ -99,9 +98,38 @@ func TestImageStudios(t *testing.T) { mockStudioReader.AssertExpectations(t) mockImageReader.AssertExpectations(t) } + + for _, test := range testTables { + mockStudioReader := &mocks.StudioReaderWriter{} + mockImageReader := &mocks.ImageReaderWriter{} + + mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() + mockStudioReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() + + doTest(mockStudioReader, mockImageReader, test) + } + + // test against aliases + const unmatchedName = "unmatched" + studio.Name.String = unmatchedName + + for _, test := range testTables { + mockStudioReader := &mocks.StudioReaderWriter{} + mockImageReader := &mocks.ImageReaderWriter{} + + mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() + mockStudioReader.On("GetAliases", studioID).Return([]string{ + studioName, + }, nil).Once() + mockStudioReader.On("GetAliases", reversedStudioID).Return([]string{}, nil).Once() + + doTest(mockStudioReader, mockImageReader, test) + } } func TestImageTags(t *testing.T) { + t.Parallel() + const imageID = 1 const tagName = "tag name" const tagID = 2 @@ -121,12 +149,7 @@ func TestImageTags(t *testing.T) { assert := assert.New(t) - for _, test := range testTables { - mockTagReader := &mocks.TagReaderWriter{} - mockImageReader := &mocks.ImageReaderWriter{} - - mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() - + doTest := func(mockTagReader *mocks.TagReaderWriter, mockImageReader *mocks.ImageReaderWriter, test pathTestTable) { if test.Matches { mockImageReader.On("GetTagIDs", imageID).Return(nil, nil).Once() mockImageReader.On("UpdateTags", imageID, []int{tagID}).Return(nil).Once() @@ -142,4 +165,31 @@ func TestImageTags(t *testing.T) { mockTagReader.AssertExpectations(t) mockImageReader.AssertExpectations(t) } + + for _, test := range testTables { + mockTagReader := &mocks.TagReaderWriter{} + mockImageReader := &mocks.ImageReaderWriter{} + + mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() + mockTagReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() + + doTest(mockTagReader, mockImageReader, test) + } + + // test against aliases + const unmatchedName = "unmatched" + tag.Name = unmatchedName + + for _, test := range testTables { + mockTagReader := &mocks.TagReaderWriter{} + mockImageReader := &mocks.ImageReaderWriter{} + + mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() + mockTagReader.On("GetAliases", tagID).Return([]string{ + tagName, + }, nil).Once() + mockTagReader.On("GetAliases", reversedTagID).Return([]string{}, nil).Once() + + doTest(mockTagReader, mockImageReader, test) + } } diff --git a/pkg/autotag/integration_test.go b/pkg/autotag/integration_test.go index 685cae74a..588124509 100644 --- a/pkg/autotag/integration_test.go +++ b/pkg/autotag/integration_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package autotag @@ -6,7 +7,6 @@ import ( "context" "database/sql" "fmt" - "io/ioutil" "os" "testing" @@ -43,7 +43,7 @@ func testTeardown(databaseFile string) { func runTests(m *testing.M) int { // create the database file - f, err := ioutil.TempFile("", "*.sqlite") + f, err := os.CreateTemp("", "*.sqlite") if err != nil { panic(fmt.Sprintf("Could not create temporary file: %s", err.Error())) } @@ -408,7 +408,12 @@ func TestParseStudioScenes(t *testing.T) { for _, s := range studios { if err := withTxn(func(r models.Repository) error { - return StudioScenes(s, nil, r.Scene()) + aliases, err := r.Studio().GetAliases(s.ID) + if err != nil { + return err + } + + return StudioScenes(s, nil, aliases, r.Scene()) }); err != nil { t.Errorf("Error auto-tagging performers: %s", err) } @@ -558,7 +563,12 @@ func TestParseStudioImages(t *testing.T) { for _, s := range studios { if err := withTxn(func(r models.Repository) error { - return StudioImages(s, nil, r.Image()) + aliases, err := r.Studio().GetAliases(s.ID) + if err != nil { + return err + } + + return StudioImages(s, nil, aliases, r.Image()) }); err != nil { t.Errorf("Error auto-tagging performers: %s", err) } @@ -708,7 +718,12 @@ func TestParseStudioGalleries(t *testing.T) { for _, s := range studios { if err := withTxn(func(r models.Repository) error { - return StudioGalleries(s, nil, r.Gallery()) + aliases, err := r.Studio().GetAliases(s.ID) + if err != nil { + return err + } + + return StudioGalleries(s, nil, aliases, r.Gallery()) }); err != nil { t.Errorf("Error auto-tagging performers: %s", err) } diff --git a/pkg/autotag/performer_test.go b/pkg/autotag/performer_test.go index 7d78b9304..3e6714ccd 100644 --- a/pkg/autotag/performer_test.go +++ b/pkg/autotag/performer_test.go @@ -9,6 +9,8 @@ import ( ) func TestPerformerScenes(t *testing.T) { + t.Parallel() + type test struct { performerName string expectedRegex string @@ -23,6 +25,10 @@ func TestPerformerScenes(t *testing.T) { "performer + name", `(?i)(?:^|_|[^\w\d])performer[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, }, + { + `performer + name\`, + `(?i)(?:^|_|[^\w\d])performer[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, + }, } for _, p := range performerNames { @@ -81,6 +87,8 @@ func testPerformerScenes(t *testing.T, performerName, expectedRegex string) { } func TestPerformerImages(t *testing.T) { + t.Parallel() + type test struct { performerName string expectedRegex string @@ -153,6 +161,8 @@ func testPerformerImages(t *testing.T, performerName, expectedRegex string) { } func TestPerformerGalleries(t *testing.T) { + t.Parallel() + type test struct { performerName string expectedRegex string diff --git a/pkg/autotag/scene_test.go b/pkg/autotag/scene_test.go index d2326522c..5e5b88806 100644 --- a/pkg/autotag/scene_test.go +++ b/pkg/autotag/scene_test.go @@ -145,6 +145,8 @@ func generateTestTable(testName, ext string) []pathTestTable { } func TestScenePerformers(t *testing.T) { + t.Parallel() + const sceneID = 1 const performerName = "performer name" const performerID = 2 @@ -188,6 +190,8 @@ func TestScenePerformers(t *testing.T) { } func TestSceneStudios(t *testing.T) { + t.Parallel() + const sceneID = 1 const studioName = "studio name" const studioID = 2 @@ -207,12 +211,7 @@ func TestSceneStudios(t *testing.T) { assert := assert.New(t) - for _, test := range testTables { - mockStudioReader := &mocks.StudioReaderWriter{} - mockSceneReader := &mocks.SceneReaderWriter{} - - mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() - + doTest := func(mockStudioReader *mocks.StudioReaderWriter, mockSceneReader *mocks.SceneReaderWriter, test pathTestTable) { if test.Matches { mockSceneReader.On("Find", sceneID).Return(&models.Scene{}, nil).Once() expectedStudioID := models.NullInt64(studioID) @@ -232,9 +231,38 @@ func TestSceneStudios(t *testing.T) { mockStudioReader.AssertExpectations(t) mockSceneReader.AssertExpectations(t) } + + for _, test := range testTables { + mockStudioReader := &mocks.StudioReaderWriter{} + mockSceneReader := &mocks.SceneReaderWriter{} + + mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() + mockStudioReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() + + doTest(mockStudioReader, mockSceneReader, test) + } + + const unmatchedName = "unmatched" + studio.Name.String = unmatchedName + + // test against aliases + for _, test := range testTables { + mockStudioReader := &mocks.StudioReaderWriter{} + mockSceneReader := &mocks.SceneReaderWriter{} + + mockStudioReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Studio{&studio, &reversedStudio}, nil).Once() + mockStudioReader.On("GetAliases", studioID).Return([]string{ + studioName, + }, nil).Once() + mockStudioReader.On("GetAliases", reversedStudioID).Return([]string{}, nil).Once() + + doTest(mockStudioReader, mockSceneReader, test) + } } func TestSceneTags(t *testing.T) { + t.Parallel() + const sceneID = 1 const tagName = "tag name" const tagID = 2 @@ -254,12 +282,7 @@ func TestSceneTags(t *testing.T) { assert := assert.New(t) - for _, test := range testTables { - mockTagReader := &mocks.TagReaderWriter{} - mockSceneReader := &mocks.SceneReaderWriter{} - - mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() - + doTest := func(mockTagReader *mocks.TagReaderWriter, mockSceneReader *mocks.SceneReaderWriter, test pathTestTable) { if test.Matches { mockSceneReader.On("GetTagIDs", sceneID).Return(nil, nil).Once() mockSceneReader.On("UpdateTags", sceneID, []int{tagID}).Return(nil).Once() @@ -275,4 +298,31 @@ func TestSceneTags(t *testing.T) { mockTagReader.AssertExpectations(t) mockSceneReader.AssertExpectations(t) } + + for _, test := range testTables { + mockTagReader := &mocks.TagReaderWriter{} + mockSceneReader := &mocks.SceneReaderWriter{} + + mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() + mockTagReader.On("GetAliases", mock.Anything).Return([]string{}, nil).Maybe() + + doTest(mockTagReader, mockSceneReader, test) + } + + const unmatchedName = "unmatched" + tag.Name = unmatchedName + + // test against aliases + for _, test := range testTables { + mockTagReader := &mocks.TagReaderWriter{} + mockSceneReader := &mocks.SceneReaderWriter{} + + mockTagReader.On("QueryForAutoTag", mock.Anything).Return([]*models.Tag{&tag, &reversedTag}, nil).Once() + mockTagReader.On("GetAliases", tagID).Return([]string{ + tagName, + }, nil).Once() + mockTagReader.On("GetAliases", reversedTagID).Return([]string{}, nil).Once() + + doTest(mockTagReader, mockSceneReader, test) + } } diff --git a/pkg/autotag/studio.go b/pkg/autotag/studio.go index ba6309c5a..1634a0fed 100644 --- a/pkg/autotag/studio.go +++ b/pkg/autotag/studio.go @@ -2,7 +2,6 @@ package autotag import ( "database/sql" - "github.com/stashapp/stash/pkg/models" ) @@ -16,7 +15,26 @@ func getMatchingStudios(path string, reader models.StudioReader) ([]*models.Stud var ret []*models.Studio for _, c := range candidates { + matches := false if nameMatchesPath(c.Name.String, path) { + matches = true + } + + if !matches { + aliases, err := reader.GetAliases(c.ID) + if err != nil { + return nil, err + } + + for _, alias := range aliases { + if nameMatchesPath(alias, path) { + matches = true + break + } + } + } + + if matches { ret = append(ret, c) } } @@ -96,37 +114,65 @@ func addGalleryStudio(galleryWriter models.GalleryReaderWriter, galleryID, studi return true, nil } -func getStudioTagger(p *models.Studio) tagger { - return tagger{ +func getStudioTagger(p *models.Studio, aliases []string) []tagger { + ret := []tagger{{ ID: p.ID, Type: "studio", Name: p.Name.String, + }} + + for _, a := range aliases { + ret = append(ret, tagger{ + ID: p.ID, + Type: "studio", + Name: a, + }) } + + return ret } // StudioScenes searches for scenes whose path matches the provided studio name and tags the scene with the studio, if studio is not already set on the scene. -func StudioScenes(p *models.Studio, paths []string, rw models.SceneReaderWriter) error { - t := getStudioTagger(p) +func StudioScenes(p *models.Studio, paths []string, aliases []string, rw models.SceneReaderWriter) error { + t := getStudioTagger(p, aliases) - return t.tagScenes(paths, rw, func(subjectID, otherID int) (bool, error) { - return addSceneStudio(rw, otherID, subjectID) - }) + for _, tt := range t { + if err := tt.tagScenes(paths, rw, func(subjectID, otherID int) (bool, error) { + return addSceneStudio(rw, otherID, subjectID) + }); err != nil { + return err + } + } + + return nil } // StudioImages searches for images whose path matches the provided studio name and tags the image with the studio, if studio is not already set on the image. -func StudioImages(p *models.Studio, paths []string, rw models.ImageReaderWriter) error { - t := getStudioTagger(p) +func StudioImages(p *models.Studio, paths []string, aliases []string, rw models.ImageReaderWriter) error { + t := getStudioTagger(p, aliases) - return t.tagImages(paths, rw, func(subjectID, otherID int) (bool, error) { - return addImageStudio(rw, otherID, subjectID) - }) + for _, tt := range t { + if err := tt.tagImages(paths, rw, func(subjectID, otherID int) (bool, error) { + return addImageStudio(rw, otherID, subjectID) + }); err != nil { + return err + } + } + + return nil } // StudioGalleries searches for galleries whose path matches the provided studio name and tags the gallery with the studio, if studio is not already set on the gallery. -func StudioGalleries(p *models.Studio, paths []string, rw models.GalleryReaderWriter) error { - t := getStudioTagger(p) +func StudioGalleries(p *models.Studio, paths []string, aliases []string, rw models.GalleryReaderWriter) error { + t := getStudioTagger(p, aliases) - return t.tagGalleries(paths, rw, func(subjectID, otherID int) (bool, error) { - return addGalleryStudio(rw, otherID, subjectID) - }) + for _, tt := range t { + if err := tt.tagGalleries(paths, rw, func(subjectID, otherID int) (bool, error) { + return addGalleryStudio(rw, otherID, subjectID) + }); err != nil { + return err + } + } + + return nil } diff --git a/pkg/autotag/studio_test.go b/pkg/autotag/studio_test.go index 886ea1361..f8c2df49e 100644 --- a/pkg/autotag/studio_test.go +++ b/pkg/autotag/studio_test.go @@ -8,35 +8,81 @@ import ( "github.com/stretchr/testify/assert" ) +type testStudioCase struct { + studioName string + expectedRegex string + aliasName string + aliasRegex string +} + +var testStudioCases = []testStudioCase{ + { + "studio name", + `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*name(?:$|_|[^\w\d])`, + "", + "", + }, + { + "studio + name", + `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, + "", + "", + }, + { + `studio + name\`, + `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, + "", + "", + }, + { + "studio name", + `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*name(?:$|_|[^\w\d])`, + "alias name", + `(?i)(?:^|_|[^\w\d])alias[.\-_ ]*name(?:$|_|[^\w\d])`, + }, + { + "studio + name", + `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, + "alias + name", + `(?i)(?:^|_|[^\w\d])alias[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, + }, + { + `studio + name\`, + `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, + `alias + name\`, + `(?i)(?:^|_|[^\w\d])alias[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, + }, +} + func TestStudioScenes(t *testing.T) { - type test struct { - studioName string - expectedRegex string - } + t.Parallel() - studioNames := []test{ - { - "studio name", - `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*name(?:$|_|[^\w\d])`, - }, - { - "studio + name", - `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, - }, - } - - for _, p := range studioNames { - testStudioScenes(t, p.studioName, p.expectedRegex) + for _, p := range testStudioCases { + testStudioScenes(t, p) } } -func testStudioScenes(t *testing.T, studioName, expectedRegex string) { +func testStudioScenes(t *testing.T, tc testStudioCase) { + studioName := tc.studioName + expectedRegex := tc.expectedRegex + aliasName := tc.aliasName + aliasRegex := tc.aliasRegex + mockSceneReader := &mocks.SceneReaderWriter{} const studioID = 2 + var aliases []string + + testPathName := studioName + if aliasName != "" { + aliases = []string{aliasName} + testPathName = aliasName + } + + matchingPaths, falsePaths := generateTestPaths(testPathName, "mp4") + var scenes []*models.Scene - matchingPaths, falsePaths := generateTestPaths(studioName, sceneExt) for i, p := range append(matchingPaths, falsePaths...) { scenes = append(scenes, &models.Scene{ ID: i + 1, @@ -64,7 +110,23 @@ func testStudioScenes(t *testing.T, studioName, expectedRegex string) { PerPage: &perPage, } - mockSceneReader.On("Query", expectedSceneFilter, expectedFindFilter).Return(scenes, len(scenes), nil).Once() + // if alias provided, then don't find by name + onNameQuery := mockSceneReader.On("Query", expectedSceneFilter, expectedFindFilter) + if aliasName == "" { + onNameQuery.Return(scenes, len(scenes), nil).Once() + } else { + onNameQuery.Return(nil, 0, nil).Once() + + expectedAliasFilter := &models.SceneFilterType{ + Organized: &organized, + Path: &models.StringCriterionInput{ + Value: aliasRegex, + Modifier: models.CriterionModifierMatchesRegex, + }, + } + + mockSceneReader.On("Query", expectedAliasFilter, expectedFindFilter).Return(scenes, len(scenes), nil).Once() + } for i := range matchingPaths { sceneID := i + 1 @@ -76,7 +138,7 @@ func testStudioScenes(t *testing.T, studioName, expectedRegex string) { }).Return(nil, nil).Once() } - err := StudioScenes(&studio, nil, mockSceneReader) + err := StudioScenes(&studio, nil, aliases, mockSceneReader) assert := assert.New(t) @@ -85,34 +147,33 @@ func testStudioScenes(t *testing.T, studioName, expectedRegex string) { } func TestStudioImages(t *testing.T) { - type test struct { - studioName string - expectedRegex string - } + t.Parallel() - studioNames := []test{ - { - "studio name", - `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*name(?:$|_|[^\w\d])`, - }, - { - "studio + name", - `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, - }, - } - - for _, p := range studioNames { - testStudioImages(t, p.studioName, p.expectedRegex) + for _, p := range testStudioCases { + testStudioImages(t, p) } } -func testStudioImages(t *testing.T, studioName, expectedRegex string) { +func testStudioImages(t *testing.T, tc testStudioCase) { + studioName := tc.studioName + expectedRegex := tc.expectedRegex + aliasName := tc.aliasName + aliasRegex := tc.aliasRegex + mockImageReader := &mocks.ImageReaderWriter{} const studioID = 2 + var aliases []string + + testPathName := studioName + if aliasName != "" { + aliases = []string{aliasName} + testPathName = aliasName + } + var images []*models.Image - matchingPaths, falsePaths := generateTestPaths(studioName, imageExt) + matchingPaths, falsePaths := generateTestPaths(testPathName, imageExt) for i, p := range append(matchingPaths, falsePaths...) { images = append(images, &models.Image{ ID: i + 1, @@ -140,7 +201,23 @@ func testStudioImages(t *testing.T, studioName, expectedRegex string) { PerPage: &perPage, } - mockImageReader.On("Query", expectedImageFilter, expectedFindFilter).Return(images, len(images), nil).Once() + // if alias provided, then don't find by name + onNameQuery := mockImageReader.On("Query", expectedImageFilter, expectedFindFilter) + if aliasName == "" { + onNameQuery.Return(images, len(images), nil).Once() + } else { + onNameQuery.Return(nil, 0, nil).Once() + + expectedAliasFilter := &models.ImageFilterType{ + Organized: &organized, + Path: &models.StringCriterionInput{ + Value: aliasRegex, + Modifier: models.CriterionModifierMatchesRegex, + }, + } + + mockImageReader.On("Query", expectedAliasFilter, expectedFindFilter).Return(images, len(images), nil).Once() + } for i := range matchingPaths { imageID := i + 1 @@ -152,7 +229,7 @@ func testStudioImages(t *testing.T, studioName, expectedRegex string) { }).Return(nil, nil).Once() } - err := StudioImages(&studio, nil, mockImageReader) + err := StudioImages(&studio, nil, aliases, mockImageReader) assert := assert.New(t) @@ -161,34 +238,32 @@ func testStudioImages(t *testing.T, studioName, expectedRegex string) { } func TestStudioGalleries(t *testing.T) { - type test struct { - studioName string - expectedRegex string - } + t.Parallel() - studioNames := []test{ - { - "studio name", - `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*name(?:$|_|[^\w\d])`, - }, - { - "studio + name", - `(?i)(?:^|_|[^\w\d])studio[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, - }, - } - - for _, p := range studioNames { - testStudioGalleries(t, p.studioName, p.expectedRegex) + for _, p := range testStudioCases { + testStudioGalleries(t, p) } } -func testStudioGalleries(t *testing.T, studioName, expectedRegex string) { +func testStudioGalleries(t *testing.T, tc testStudioCase) { + studioName := tc.studioName + expectedRegex := tc.expectedRegex + aliasName := tc.aliasName + aliasRegex := tc.aliasRegex mockGalleryReader := &mocks.GalleryReaderWriter{} const studioID = 2 + var aliases []string + + testPathName := studioName + if aliasName != "" { + aliases = []string{aliasName} + testPathName = aliasName + } + var galleries []*models.Gallery - matchingPaths, falsePaths := generateTestPaths(studioName, galleryExt) + matchingPaths, falsePaths := generateTestPaths(testPathName, galleryExt) for i, p := range append(matchingPaths, falsePaths...) { galleries = append(galleries, &models.Gallery{ ID: i + 1, @@ -216,7 +291,23 @@ func testStudioGalleries(t *testing.T, studioName, expectedRegex string) { PerPage: &perPage, } - mockGalleryReader.On("Query", expectedGalleryFilter, expectedFindFilter).Return(galleries, len(galleries), nil).Once() + // if alias provided, then don't find by name + onNameQuery := mockGalleryReader.On("Query", expectedGalleryFilter, expectedFindFilter) + if aliasName == "" { + onNameQuery.Return(galleries, len(galleries), nil).Once() + } else { + onNameQuery.Return(nil, 0, nil).Once() + + expectedAliasFilter := &models.GalleryFilterType{ + Organized: &organized, + Path: &models.StringCriterionInput{ + Value: aliasRegex, + Modifier: models.CriterionModifierMatchesRegex, + }, + } + + mockGalleryReader.On("Query", expectedAliasFilter, expectedFindFilter).Return(galleries, len(galleries), nil).Once() + } for i := range matchingPaths { galleryID := i + 1 @@ -228,7 +319,7 @@ func testStudioGalleries(t *testing.T, studioName, expectedRegex string) { }).Return(nil, nil).Once() } - err := StudioGalleries(&studio, nil, mockGalleryReader) + err := StudioGalleries(&studio, nil, aliases, mockGalleryReader) assert := assert.New(t) diff --git a/pkg/autotag/tag.go b/pkg/autotag/tag.go index c1d2cf271..48de81417 100644 --- a/pkg/autotag/tag.go +++ b/pkg/autotag/tag.go @@ -16,9 +16,27 @@ func getMatchingTags(path string, tagReader models.TagReader) ([]*models.Tag, er } var ret []*models.Tag - for _, p := range tags { - if nameMatchesPath(p.Name, path) { - ret = append(ret, p) + for _, t := range tags { + matches := false + if nameMatchesPath(t.Name, path) { + matches = true + } + + if !matches { + aliases, err := tagReader.GetAliases(t.ID) + if err != nil { + return nil, err + } + for _, alias := range aliases { + if nameMatchesPath(alias, path) { + matches = true + break + } + } + } + + if matches { + ret = append(ret, t) } } diff --git a/pkg/autotag/tag_test.go b/pkg/autotag/tag_test.go index f23cab47b..07a85856e 100644 --- a/pkg/autotag/tag_test.go +++ b/pkg/autotag/tag_test.go @@ -28,6 +28,12 @@ var testTagCases = []testTagCase{ "", "", }, + { + `tag + name\`, + `(?i)(?:^|_|[^\w\d])tag[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, + "", + "", + }, { "tag name", `(?i)(?:^|_|[^\w\d])tag[.\-_ ]*name(?:$|_|[^\w\d])`, @@ -40,9 +46,17 @@ var testTagCases = []testTagCase{ "alias + name", `(?i)(?:^|_|[^\w\d])alias[.\-_ ]*\+[.\-_ ]*name(?:$|_|[^\w\d])`, }, + { + `tag + name\`, + `(?i)(?:^|_|[^\w\d])tag[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, + `alias + name\`, + `(?i)(?:^|_|[^\w\d])alias[.\-_ ]*\+[.\-_ ]*name\\(?:$|_|[^\w\d])`, + }, } func TestTagScenes(t *testing.T) { + t.Parallel() + for _, p := range testTagCases { testTagScenes(t, p) } @@ -129,6 +143,8 @@ func testTagScenes(t *testing.T, tc testTagCase) { } func TestTagImages(t *testing.T) { + t.Parallel() + for _, p := range testTagCases { testTagImages(t, p) } @@ -214,6 +230,8 @@ func testTagImages(t *testing.T, tc testTagCase) { } func TestTagGalleries(t *testing.T) { + t.Parallel() + for _, p := range testTagCases { testTagGalleries(t, p) } diff --git a/pkg/autotag/tagger.go b/pkg/autotag/tagger.go index 4bca225c0..c0555d401 100644 --- a/pkg/autotag/tagger.go +++ b/pkg/autotag/tagger.go @@ -25,22 +25,9 @@ import ( const separatorChars = `.\-_ ` -// fixes #1292 -func escapePathRegex(name string) string { - ret := name - - chars := `+*?()|[]{}^$` - for _, c := range chars { - cStr := string(c) - ret = strings.ReplaceAll(ret, cStr, `\`+cStr) - } - - return ret -} - func getPathQueryRegex(name string) string { // escape specific regex characters - name = escapePathRegex(name) + name = regexp.QuoteMeta(name) // handle path separators const separator = `[` + separatorChars + `]` @@ -52,7 +39,7 @@ func getPathQueryRegex(name string) string { func nameMatchesPath(name, path string) bool { // escape specific regex characters - name = escapePathRegex(name) + name = regexp.QuoteMeta(name) name = strings.ToLower(name) path = strings.ToLower(path) diff --git a/pkg/database/custom_migrations.go b/pkg/database/custom_migrations.go new file mode 100644 index 000000000..9e343a515 --- /dev/null +++ b/pkg/database/custom_migrations.go @@ -0,0 +1,72 @@ +package database + +import ( + "database/sql" + "fmt" + "strings" + + "github.com/jmoiron/sqlx" + "github.com/stashapp/stash/pkg/logger" +) + +func runCustomMigrations() error { + if err := createImagesChecksumIndex(); err != nil { + return err + } + + return nil +} + +func createImagesChecksumIndex() error { + return WithTxn(func(tx *sqlx.Tx) error { + row := tx.QueryRow("SELECT 1 AS found FROM sqlite_master WHERE type = 'index' AND name = 'images_checksum_unique'") + err := row.Err() + if err != nil && err != sql.ErrNoRows { + return err + } + + if err == nil { + var found bool + if err := row.Scan(&found); err != nil && err != sql.ErrNoRows { + return fmt.Errorf("error while scanning for index: %w", err) + } + if found { + return nil + } + } + + _, err = tx.Exec("CREATE UNIQUE INDEX images_checksum_unique ON images (checksum)") + if err == nil { + _, err = tx.Exec("DROP INDEX IF EXISTS index_images_checksum") + if err != nil { + logger.Errorf("Failed to remove surrogate images.checksum index: %s", err) + } + logger.Info("Created unique constraint on images table") + return nil + } + + _, err = tx.Exec("CREATE INDEX IF NOT EXISTS index_images_checksum ON images (checksum)") + if err != nil { + logger.Errorf("Unable to create index on images.checksum: %s", err) + } + + var result []struct { + Checksum string `db:"checksum"` + } + + err = tx.Select(&result, "SELECT checksum FROM images GROUP BY checksum HAVING COUNT(1) > 1") + if err != nil && err != sql.ErrNoRows { + logger.Errorf("Unable to determine non-unique image checksums: %s", err) + return nil + } + + checksums := make([]string, len(result)) + for i, res := range result { + checksums[i] = res.Checksum + } + + logger.Warnf("The following duplicate image checksums have been found. Please remove the duplicates and restart. %s", strings.Join(checksums, ", ")) + + return nil + }) +} diff --git a/pkg/database/database.go b/pkg/database/database.go index c992601d8..f84207b65 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -2,6 +2,7 @@ package database import ( "database/sql" + "embed" "errors" "fmt" "os" @@ -9,10 +10,9 @@ import ( "time" "github.com/fvbommel/sortorder" - "github.com/gobuffalo/packr/v2" "github.com/golang-migrate/migrate/v4" sqlite3mig "github.com/golang-migrate/migrate/v4/database/sqlite3" - "github.com/golang-migrate/migrate/v4/source" + "github.com/golang-migrate/migrate/v4/source/iofs" "github.com/jmoiron/sqlx" sqlite3 "github.com/mattn/go-sqlite3" @@ -21,11 +21,14 @@ import ( ) var DB *sqlx.DB -var WriteMu *sync.Mutex +var WriteMu sync.Mutex var dbPath string -var appSchemaVersion uint = 25 +var appSchemaVersion uint = 28 var databaseSchemaVersion uint +//go:embed migrations/*.sql +var migrationsBox embed.FS + var ( // ErrMigrationNeeded indicates that a database migration is needed // before the database can be initialized @@ -84,14 +87,32 @@ func Initialize(databasePath string) error { const disableForeignKeys = false DB = open(databasePath, disableForeignKeys) - WriteMu = &sync.Mutex{} + + if err := runCustomMigrations(); err != nil { + return err + } + + return nil +} + +func Close() error { + WriteMu.Lock() + defer WriteMu.Unlock() + + if DB != nil { + if err := DB.Close(); err != nil { + return err + } + + DB = nil + } return nil } func open(databasePath string, disableForeignKeys bool) *sqlx.DB { // https://github.com/mattn/go-sqlite3 - url := "file:" + databasePath + "?_journal=WAL" + url := "file:" + databasePath + "?_journal=WAL&_sync=NORMAL" if !disableForeignKeys { url += "&_fk=true" } @@ -130,7 +151,10 @@ func Reset(databasePath string) error { } } - Initialize(databasePath) + if err := Initialize(databasePath); err != nil { + return fmt.Errorf("[reset DB] unable to initialize: %w", err) + } + return nil } @@ -149,7 +173,7 @@ func Backup(db *sqlx.DB, backupPath string) error { logger.Infof("Backing up database into: %s", backupPath) _, err := db.Exec(`VACUUM INTO "` + backupPath + `"`) if err != nil { - return fmt.Errorf("Vacuum failed: %s", err) + return fmt.Errorf("vacuum failed: %s", err) } return nil @@ -182,17 +206,13 @@ func Version() uint { } func getMigrate() (*migrate.Migrate, error) { - migrationsBox := packr.New("Migrations Box", "./migrations") - packrSource := &Packr2Source{ - Box: migrationsBox, - Migrations: source.NewMigrations(), + migrations, err := iofs.New(migrationsBox, "migrations") + if err != nil { + panic(err.Error()) } - databasePath := utils.FixWindowsPath(dbPath) - s, _ := WithInstance(packrSource) - const disableForeignKeys = true - conn := open(databasePath, disableForeignKeys) + conn := open(dbPath, disableForeignKeys) driver, err := sqlite3mig.WithInstance(conn.DB, &sqlite3mig.Config{}) if err != nil { @@ -201,9 +221,9 @@ func getMigrate() (*migrate.Migrate, error) { // use sqlite3Driver so that migration has access to durationToTinyInt return migrate.NewWithInstance( - "packr2", - s, - databasePath, + "iofs", + migrations, + dbPath, driver, ) } @@ -225,6 +245,7 @@ func RunMigrations() error { if err != nil { panic(err.Error()) } + defer m.Close() databaseSchemaVersion, _, _ = m.Version() stepNumber := appSchemaVersion - databaseSchemaVersion @@ -233,22 +254,20 @@ func RunMigrations() error { err = m.Steps(int(stepNumber)) if err != nil { // migration failed - logger.Errorf("Error migrating database: %s", err.Error()) - m.Close() return err } } - m.Close() - // re-initialise the database - Initialize(dbPath) + if err = Initialize(dbPath); err != nil { + logger.Warnf("Error re-initializing the database: %v", err) + } // run a vacuum on the database logger.Info("Performing vacuum on database") _, err = DB.Exec("VACUUM") if err != nil { - logger.Warnf("error while performing post-migration vacuum: %s", err.Error()) + logger.Warnf("error while performing post-migration vacuum: %v", err) } return nil @@ -265,7 +284,7 @@ func registerCustomDriver() { for name, fn := range funcs { if err := conn.RegisterFunc(name, fn, true); err != nil { - return fmt.Errorf("Error registering function %s: %s", name, err.Error()) + return fmt.Errorf("error registering function %s: %s", name, err.Error()) } } @@ -279,7 +298,7 @@ func registerCustomDriver() { }) if err != nil { - return fmt.Errorf("Error registering natural sort collation: %s", err.Error()) + return fmt.Errorf("error registering natural sort collation: %s", err.Error()) } return nil diff --git a/pkg/database/migrations/26_tag_hierarchy.up.sql b/pkg/database/migrations/26_tag_hierarchy.up.sql new file mode 100644 index 000000000..6e9855fc2 --- /dev/null +++ b/pkg/database/migrations/26_tag_hierarchy.up.sql @@ -0,0 +1,7 @@ +CREATE TABLE tags_relations ( + parent_id integer, + child_id integer, + primary key (parent_id, child_id), + foreign key (parent_id) references tags(id) on delete cascade, + foreign key (child_id) references tags(id) on delete cascade +); diff --git a/pkg/database/migrations/27_studio_aliases.up.sql b/pkg/database/migrations/27_studio_aliases.up.sql new file mode 100644 index 000000000..a7be876b9 --- /dev/null +++ b/pkg/database/migrations/27_studio_aliases.up.sql @@ -0,0 +1,7 @@ +CREATE TABLE `studio_aliases` ( + `studio_id` integer, + `alias` varchar(255) NOT NULL, + foreign key(`studio_id`) references `studios`(`id`) on delete CASCADE +); + +CREATE UNIQUE INDEX `studio_aliases_alias_unique` on `studio_aliases` (`alias`); diff --git a/pkg/database/migrations/28_images_indexes.up.sql b/pkg/database/migrations/28_images_indexes.up.sql new file mode 100644 index 000000000..1fbb1cfe2 --- /dev/null +++ b/pkg/database/migrations/28_images_indexes.up.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS `images_path_unique`; + +CREATE UNIQUE INDEX `images_path_unique` ON `images` (`path`); diff --git a/pkg/database/packr_source.go b/pkg/database/packr_source.go deleted file mode 100644 index 284e33bf6..000000000 --- a/pkg/database/packr_source.go +++ /dev/null @@ -1,92 +0,0 @@ -package database - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - - "github.com/gobuffalo/packr/v2" - "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/source" -) - -type Packr2Source struct { - Box *packr.Box - Migrations *source.Migrations -} - -func init() { - source.Register("packr2", &Packr2Source{}) -} - -func WithInstance(instance *Packr2Source) (source.Driver, error) { - for _, fi := range instance.Box.List() { - m, err := source.DefaultParse(fi) - if err != nil { - continue // ignore files that we can't parse - } - - if !instance.Migrations.Append(m) { - return nil, fmt.Errorf("unable to parse file %v", fi) - } - } - - return instance, nil -} - -func (s *Packr2Source) Open(url string) (source.Driver, error) { - return nil, fmt.Errorf("not implemented") -} - -func (s *Packr2Source) Close() error { - s.Migrations = nil - return nil -} - -func (s *Packr2Source) First() (version uint, err error) { - if v, ok := s.Migrations.First(); !ok { - return 0, os.ErrNotExist - } else { - return v, nil - } -} - -func (s *Packr2Source) Prev(version uint) (prevVersion uint, err error) { - if v, ok := s.Migrations.Prev(version); !ok { - return 0, os.ErrNotExist - } else { - return v, nil - } -} - -func (s *Packr2Source) Next(version uint) (nextVersion uint, err error) { - if v, ok := s.Migrations.Next(version); !ok { - return 0, os.ErrNotExist - } else { - return v, nil - } -} - -func (s *Packr2Source) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { - if migration, ok := s.Migrations.Up(version); !ok { - return nil, "", os.ErrNotExist - } else { - b, _ := s.Box.Find(migration.Raw) - return ioutil.NopCloser(bytes.NewBuffer(b)), - migration.Identifier, - nil - } -} - -func (s *Packr2Source) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { - if migration, ok := s.Migrations.Down(version); !ok { - return nil, "", migrate.ErrNilVersion - } else { - b, _ := s.Box.Find(migration.Raw) - return ioutil.NopCloser(bytes.NewBuffer(b)), - migration.Identifier, - nil - } -} diff --git a/pkg/database/transaction.go b/pkg/database/transaction.go index c3ef9a85f..32c1ab171 100644 --- a/pkg/database/transaction.go +++ b/pkg/database/transaction.go @@ -4,6 +4,7 @@ import ( "context" "github.com/jmoiron/sqlx" + "github.com/stashapp/stash/pkg/logger" ) // WithTxn executes the provided function within a transaction. It rolls back @@ -17,11 +18,15 @@ func WithTxn(fn func(tx *sqlx.Tx) error) error { defer func() { if p := recover(); p != nil { // a panic occurred, rollback and repanic - tx.Rollback() + if err := tx.Rollback(); err != nil { + logger.Warnf("failure when performing transaction rollback: %v", err) + } panic(p) } else if err != nil { // something went wrong, rollback - tx.Rollback() + if err := tx.Rollback(); err != nil { + logger.Warnf("failure when performing transaction rollback: %v", err) + } } else { // all good, commit err = tx.Commit() diff --git a/pkg/dlna/cds.go b/pkg/dlna/cds.go index e99f63be8..d23660d0b 100644 --- a/pkg/dlna/cds.go +++ b/pkg/dlna/cds.go @@ -533,7 +533,6 @@ func (me *contentDirectoryService) getStudioScenes(paths []string, host string) Studios: &models.HierarchicalMultiCriterionInput{ Modifier: models.CriterionModifierIncludes, Value: []string{paths[0]}, - Depth: 0, }, } @@ -570,7 +569,7 @@ func (me *contentDirectoryService) getTags() []interface{} { func (me *contentDirectoryService) getTagScenes(paths []string, host string) []interface{} { sceneFilter := &models.SceneFilterType{ - Tags: &models.MultiCriterionInput{ + Tags: &models.HierarchicalMultiCriterionInput{ Modifier: models.CriterionModifierIncludes, Value: []string{paths[0]}, }, diff --git a/pkg/dlna/dms.go b/pkg/dlna/dms.go index 85c25c979..d7c5efa85 100644 --- a/pkg/dlna/dms.go +++ b/pkg/dlna/dms.go @@ -33,7 +33,6 @@ import ( "encoding/xml" "fmt" "io" - "io/ioutil" "net" "net/http" "net/http/pprof" @@ -58,7 +57,6 @@ const ( resPath = "/res" iconPath = "/icon" rootDescPath = "/rootDesc.xml" - contentDirectorySCPDURL = "/scpd/ContentDirectory.xml" contentDirectoryEventSubURL = "/evt/ContentDirectory" serviceControlURL = "/ctl" deviceIconPath = "/deviceIcon" @@ -417,7 +415,7 @@ func (me *Server) serveIcon(w http.ResponseWriter, r *http.Request) { } var scene *models.Scene - me.txnManager.WithReadTxn(context.Background(), func(r models.ReaderRepository) error { + err := me.txnManager.WithReadTxn(context.Background(), func(r models.ReaderRepository) error { idInt, err := strconv.Atoi(sceneId) if err != nil { return nil @@ -425,6 +423,9 @@ func (me *Server) serveIcon(w http.ResponseWriter, r *http.Request) { scene, _ = r.Scene().Find(idInt) return nil }) + if err != nil { + logger.Warnf("failed to execute read transaction while trying to serve an icon: %v", err) + } if scene == nil { return @@ -481,7 +482,7 @@ func (me *Server) contentDirectoryInitialEvent(urls []*url.URL, sid string) { logger.Errorf("Could not notify %s: %s", _url.String(), err) continue } - b, _ := ioutil.ReadAll(resp.Body) + b, _ := io.ReadAll(resp.Body) if len(b) > 0 { logger.Debug(string(b)) } @@ -553,7 +554,7 @@ func (me *Server) initMux(mux *http.ServeMux) { mux.HandleFunc(resPath, func(w http.ResponseWriter, r *http.Request) { sceneId := r.URL.Query().Get("scene") var scene *models.Scene - me.txnManager.WithReadTxn(context.Background(), func(r models.ReaderRepository) error { + err := me.txnManager.WithReadTxn(context.Background(), func(r models.ReaderRepository) error { sceneIdInt, err := strconv.Atoi(sceneId) if err != nil { return nil @@ -561,6 +562,9 @@ func (me *Server) initMux(mux *http.ServeMux) { scene, _ = r.Scene().Find(sceneIdInt) return nil }) + if err != nil { + logger.Warnf("failed to execute read transaction for scene id (%v): %v", sceneId, err) + } if scene == nil { return @@ -572,7 +576,9 @@ func (me *Server) initMux(mux *http.ServeMux) { w.Header().Set("content-type", `text/xml; charset="utf-8"`) w.Header().Set("content-length", fmt.Sprint(len(me.rootDescXML))) w.Header().Set("server", serverField) - w.Write(me.rootDescXML) + if k, err := w.Write(me.rootDescXML); err != nil { + logger.Warnf("could not write rootDescXML (wrote %v bytes of %v): %v", k, len(me.rootDescXML), err) + } }) handleSCPDs(mux) mux.HandleFunc(serviceControlURL, me.serviceControlHandler) diff --git a/pkg/dlna/service.go b/pkg/dlna/service.go index f0fb759e4..d0b3ba196 100644 --- a/pkg/dlna/service.go +++ b/pkg/dlna/service.go @@ -251,7 +251,9 @@ func (s *Service) Stop(duration *time.Duration) { if s.startTimer == nil { s.startTimer = time.AfterFunc(*duration, func() { - s.Start(nil) + if err := s.Start(nil); err != nil { + logger.Warnf("error restarting DLNA server: %v", err) + } }) t := time.Now().Add(*duration) s.startTime = &t diff --git a/pkg/ffmpeg/encoder.go b/pkg/ffmpeg/encoder.go index 9d96dadf3..ed9c6e31f 100644 --- a/pkg/ffmpeg/encoder.go +++ b/pkg/ffmpeg/encoder.go @@ -2,7 +2,7 @@ package ffmpeg import ( "bytes" - "io/ioutil" + "io" "os" "os/exec" "strings" @@ -63,7 +63,9 @@ func KillRunningEncoders(path string) { for _, process := range processes { // assume it worked, don't check for error logger.Infof("Killing encoder process for file: %s", path) - process.Kill() + if err := process.Kill(); err != nil { + logger.Warnf("failed to kill process %v: %v", process.Pid, err) + } // wait for the process to die before returning // don't wait more than a few seconds @@ -124,7 +126,7 @@ func (e *Encoder) runTranscode(probeResult VideoFile, args []string) (string, er } } - stdoutData, _ := ioutil.ReadAll(stdout) + stdoutData, _ := io.ReadAll(stdout) stdoutString := string(stdoutData) registerRunningEncoder(probeResult.Path, cmd.Process) diff --git a/pkg/ffmpeg/ffprobe.go b/pkg/ffmpeg/ffprobe.go index 2dc1e2ad2..ed276f14a 100644 --- a/pkg/ffmpeg/ffprobe.go +++ b/pkg/ffmpeg/ffprobe.go @@ -236,7 +236,7 @@ func NewVideoFile(ffprobePath string, videoPath string, stripExt bool) (*VideoFi probeJSON := &FFProbeJSON{} if err := json.Unmarshal(out, probeJSON); err != nil { - return nil, fmt.Errorf("Error unmarshalling video data for <%s>: %s", videoPath, err.Error()) + return nil, fmt.Errorf("error unmarshalling video data for <%s>: %s", videoPath, err.Error()) } return parse(videoPath, probeJSON, stripExt) @@ -273,8 +273,9 @@ func parse(filePath string, probeJSON *FFProbeJSON, stripExt bool) (*VideoFile, result.Duration = math.Round(duration*100) / 100 fileStat, err := os.Stat(filePath) if err != nil { - logger.Errorf("Error statting file <%s>: %s", filePath, err.Error()) - return nil, err + statErr := fmt.Errorf("error statting file <%s>: %w", filePath, err) + logger.Errorf("%v", statErr) + return nil, statErr } result.Size = fileStat.Size() result.StartTime, _ = strconv.ParseFloat(probeJSON.Format.StartTime, 64) diff --git a/pkg/ffmpeg/hls.go b/pkg/ffmpeg/hls.go index 4ac5788ec..f0f6b5205 100644 --- a/pkg/ffmpeg/hls.go +++ b/pkg/ffmpeg/hls.go @@ -21,9 +21,8 @@ func WriteHLSPlaylist(probeResult VideoFile, baseUrl string, w io.Writer) { leftover := duration upTo := 0.0 - tsURL := baseUrl i := strings.LastIndex(baseUrl, ".m3u8") - tsURL = baseUrl[0:i] + ".ts" + tsURL := baseUrl[0:i] + ".ts" for leftover > 0 { thisLength := hlsSegmentLength diff --git a/pkg/ffmpeg/stream.go b/pkg/ffmpeg/stream.go index 420667680..f99c3c95c 100644 --- a/pkg/ffmpeg/stream.go +++ b/pkg/ffmpeg/stream.go @@ -2,7 +2,6 @@ package ffmpeg import ( "io" - "io/ioutil" "net/http" "os" "os/exec" @@ -32,7 +31,9 @@ func (s *Stream) Serve(w http.ResponseWriter, r *http.Request) { notify := r.Context().Done() go func() { <-notify - s.Process.Kill() + if err := s.Process.Kill(); err != nil { + logger.Warnf("unable to kill os process %v: %v", s.Process.Pid, err) + } }() _, err := io.Copy(w, s.Stdout) @@ -224,11 +225,15 @@ func (e *Encoder) stream(probeResult VideoFile, options TranscodeStreamOptions) } registerRunningEncoder(probeResult.Path, cmd.Process) - go waitAndDeregister(probeResult.Path, cmd) + go func() { + if err := waitAndDeregister(probeResult.Path, cmd); err != nil { + logger.Warnf("Error while deregistering ffmpeg stream: %v", err) + } + }() // stderr must be consumed or the process deadlocks go func() { - stderrData, _ := ioutil.ReadAll(stderr) + stderrData, _ := io.ReadAll(stderr) stderrString := string(stderrData) if len(stderrString) > 0 { logger.Debugf("[stream] ffmpeg stderr: %s", stderrString) diff --git a/pkg/gallery/export_test.go b/pkg/gallery/export_test.go index 56b3e540a..ac46871fa 100644 --- a/pkg/gallery/export_test.go +++ b/pkg/gallery/export_test.go @@ -19,7 +19,7 @@ const ( missingStudioID = 5 errStudioID = 6 - noTagsID = 11 + // noTagsID = 11 errTagsID = 12 ) @@ -39,13 +39,10 @@ const ( studioName = "studioName" ) -var names = []string{ - "name1", - "name2", -} - -var createTime time.Time = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) -var updateTime time.Time = time.Date(2002, 01, 01, 0, 0, 0, 0, time.UTC) +var ( + createTime = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) + updateTime = time.Date(2002, 01, 01, 0, 0, 0, 0, time.UTC) +) func createFullGallery(id int) models.Gallery { return models.Gallery{ @@ -71,18 +68,6 @@ func createFullGallery(id int) models.Gallery { } } -func createEmptyGallery(id int) models.Gallery { - return models.Gallery{ - ID: id, - CreatedAt: models.SQLiteTimestamp{ - Timestamp: createTime, - }, - UpdatedAt: models.SQLiteTimestamp{ - Timestamp: updateTime, - }, - } -} - func createFullJSONGallery() *jsonschema.Gallery { return &jsonschema.Gallery{ Title: title, @@ -103,17 +88,6 @@ func createFullJSONGallery() *jsonschema.Gallery { } } -func createEmptyJSONGallery() *jsonschema.Gallery { - return &jsonschema.Gallery{ - CreatedAt: models.JSONTime{ - Time: createTime, - }, - UpdatedAt: models.JSONTime{ - Time: updateTime, - }, - } -} - type basicTestScenario struct { input models.Gallery expected *jsonschema.Gallery diff --git a/pkg/gallery/import_test.go b/pkg/gallery/import_test.go index 19de14f35..962bd18d7 100644 --- a/pkg/gallery/import_test.go +++ b/pkg/gallery/import_test.go @@ -13,8 +13,8 @@ import ( ) const ( - galleryNameErr = "galleryNameErr" - existingGalleryName = "existingGalleryName" + galleryNameErr = "galleryNameErr" + // existingGalleryName = "existingGalleryName" existingGalleryID = 100 existingStudioID = 101 @@ -39,8 +39,10 @@ const ( errChecksum = "errChecksum" ) -var createdAt time.Time = time.Date(2001, time.January, 2, 1, 2, 3, 4, time.Local) -var updatedAt time.Time = time.Date(2002, time.January, 2, 1, 2, 3, 4, time.Local) +var ( + createdAt = time.Date(2001, time.January, 2, 1, 2, 3, 4, time.Local) + updatedAt = time.Date(2002, time.January, 2, 1, 2, 3, 4, time.Local) +) func TestImporterName(t *testing.T) { i := Importer{ diff --git a/pkg/gallery/query.go b/pkg/gallery/query.go index a354f5aa5..f15e480f2 100644 --- a/pkg/gallery/query.go +++ b/pkg/gallery/query.go @@ -22,7 +22,6 @@ func CountByStudioID(r models.GalleryReader, id int) (int, error) { Studios: &models.HierarchicalMultiCriterionInput{ Value: []string{strconv.Itoa(id)}, Modifier: models.CriterionModifierIncludes, - Depth: 0, }, } @@ -31,7 +30,7 @@ func CountByStudioID(r models.GalleryReader, id int) (int, error) { func CountByTagID(r models.GalleryReader, id int) (int, error) { filter := &models.GalleryFilterType{ - Tags: &models.MultiCriterionInput{ + Tags: &models.HierarchicalMultiCriterionInput{ Value: []string{strconv.Itoa(id)}, Modifier: models.CriterionModifierIncludes, }, diff --git a/pkg/image/export_test.go b/pkg/image/export_test.go index 9c7ab5ed1..1e0a7d411 100644 --- a/pkg/image/export_test.go +++ b/pkg/image/export_test.go @@ -13,8 +13,8 @@ import ( ) const ( - imageID = 1 - noImageID = 2 + imageID = 1 + // noImageID = 2 errImageID = 3 studioID = 4 @@ -24,17 +24,17 @@ const ( // noGalleryID = 7 // errGalleryID = 8 - noTagsID = 11 + // noTagsID = 11 errTagsID = 12 - noMoviesID = 13 - errMoviesID = 14 - errFindMovieID = 15 + // noMoviesID = 13 + // errMoviesID = 14 + // errFindMovieID = 15 - noMarkersID = 16 - errMarkersID = 17 - errFindPrimaryTagID = 18 - errFindByMarkerID = 19 + // noMarkersID = 16 + // errMarkersID = 17 + // errFindPrimaryTagID = 18 + // errFindByMarkerID = 19 ) const ( @@ -53,13 +53,10 @@ const ( //galleryChecksum = "galleryChecksum" ) -var names = []string{ - "name1", - "name2", -} - -var createTime time.Time = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) -var updateTime time.Time = time.Date(2002, 01, 01, 0, 0, 0, 0, time.UTC) +var ( + createTime = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) + updateTime = time.Date(2002, 01, 01, 0, 0, 0, 0, time.UTC) +) func createFullImage(id int) models.Image { return models.Image{ @@ -81,18 +78,6 @@ func createFullImage(id int) models.Image { } } -func createEmptyImage(id int) models.Image { - return models.Image{ - ID: id, - CreatedAt: models.SQLiteTimestamp{ - Timestamp: createTime, - }, - UpdatedAt: models.SQLiteTimestamp{ - Timestamp: updateTime, - }, - } -} - func createFullJSONImage() *jsonschema.Image { return &jsonschema.Image{ Title: title, @@ -114,18 +99,6 @@ func createFullJSONImage() *jsonschema.Image { } } -func createEmptyJSONImage() *jsonschema.Image { - return &jsonschema.Image{ - File: &jsonschema.ImageFile{}, - CreatedAt: models.JSONTime{ - Time: createTime, - }, - UpdatedAt: models.JSONTime{ - Time: updateTime, - }, - } -} - type basicTestScenario struct { input models.Image expected *jsonschema.Image diff --git a/pkg/image/image.go b/pkg/image/image.go index 64b766e6d..486c65ef4 100644 --- a/pkg/image/image.go +++ b/pkg/image/image.go @@ -6,13 +6,13 @@ import ( "fmt" "image" "io" - "io/ioutil" "net/http" "os" "path/filepath" "strings" "time" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" _ "golang.org/x/image/webp" @@ -35,6 +35,18 @@ func GetSourceImage(i *models.Image) (image.Image, error) { return srcImage, nil } +func DecodeSourceImage(i *models.Image) (*image.Config, *string, error) { + f, err := openSourceImage(i.Path) + if err != nil { + return nil, nil, err + } + defer f.Close() + + config, format, err := image.DecodeConfig(f) + + return &config, &format, err +} + func CalculateMD5(path string) (string, error) { f, err := openSourceImage(path) if err != nil { @@ -154,15 +166,15 @@ func SetFileDetails(i *models.Image) error { return err } - src, _ := GetSourceImage(i) + config, _, err := DecodeSourceImage(i) - if src != nil { + if err == nil { i.Width = sql.NullInt64{ - Int64: int64(src.Bounds().Max.X), + Int64: int64(config.Width), Valid: true, } i.Height = sql.NullInt64{ - Int64: int64(src.Bounds().Max.Y), + Int64: int64(config.Height), Valid: true, } } @@ -234,13 +246,15 @@ func Serve(w http.ResponseWriter, r *http.Request, path string) { } defer rc.Close() - data, err := ioutil.ReadAll(rc) + data, err := io.ReadAll(rc) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - w.Write(data) + if k, err := w.Write(data); err != nil { + logger.Warnf("failure while serving image (wrote %v bytes out of %v): %v", k, len(data), err) + } } } diff --git a/pkg/image/import_test.go b/pkg/image/import_test.go index 91978dac8..8b7099823 100644 --- a/pkg/image/import_test.go +++ b/pkg/image/import_test.go @@ -11,20 +11,18 @@ import ( "github.com/stretchr/testify/mock" ) -const invalidImage = "aW1hZ2VCeXRlcw&&" - const ( path = "path" - imageNameErr = "imageNameErr" - existingImageName = "existingImageName" + imageNameErr = "imageNameErr" + // existingImageName = "existingImageName" existingImageID = 100 existingStudioID = 101 existingGalleryID = 102 existingPerformerID = 103 - existingMovieID = 104 - existingTagID = 105 + // existingMovieID = 104 + existingTagID = 105 existingStudioName = "existingStudioName" existingStudioErr = "existingStudioErr" diff --git a/pkg/image/query.go b/pkg/image/query.go index b5c70738c..7b2dac990 100644 --- a/pkg/image/query.go +++ b/pkg/image/query.go @@ -22,7 +22,6 @@ func CountByStudioID(r models.ImageReader, id int) (int, error) { Studios: &models.HierarchicalMultiCriterionInput{ Value: []string{strconv.Itoa(id)}, Modifier: models.CriterionModifierIncludes, - Depth: 0, }, } @@ -31,7 +30,7 @@ func CountByStudioID(r models.ImageReader, id int) (int, error) { func CountByTagID(r models.ImageReader, id int) (int, error) { filter := &models.ImageFilterType{ - Tags: &models.MultiCriterionInput{ + Tags: &models.HierarchicalMultiCriterionInput{ Value: []string{strconv.Itoa(id)}, Modifier: models.CriterionModifierIncludes, }, diff --git a/pkg/image/thumbnail.go b/pkg/image/thumbnail.go index 107b77143..bb4cac743 100644 --- a/pkg/image/thumbnail.go +++ b/pkg/image/thumbnail.go @@ -2,39 +2,130 @@ package image import ( "bytes" - "image" - "image/jpeg" + "errors" + "fmt" + "os/exec" + "runtime" + "strings" + "sync" - "github.com/disintegration/imaging" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" ) -func ThumbnailNeeded(srcImage image.Image, maxSize int) bool { - dim := srcImage.Bounds().Max - w := dim.X - h := dim.Y +var vipsPath string +var once sync.Once - return w > maxSize || h > maxSize +type ThumbnailEncoder struct { + FFMPEGPath string + VipsPath string +} + +func GetVipsPath() string { + once.Do(func() { + vipsPath, _ = exec.LookPath("vips") + }) + return vipsPath +} + +func NewThumbnailEncoder(ffmpegPath string) ThumbnailEncoder { + return ThumbnailEncoder{ + FFMPEGPath: ffmpegPath, + VipsPath: GetVipsPath(), + } } // GetThumbnail returns the thumbnail image of the provided image resized to // the provided max size. It resizes based on the largest X/Y direction. // It returns nil and an error if an error occurs reading, decoding or encoding // the image. -func GetThumbnail(srcImage image.Image, maxSize int) ([]byte, error) { - var resizedImage image.Image - - // if height is longer then resize by height instead of width - dim := srcImage.Bounds().Max - if dim.Y > dim.X { - resizedImage = imaging.Resize(srcImage, 0, maxSize, imaging.Box) - } else { - resizedImage = imaging.Resize(srcImage, maxSize, 0, imaging.Box) - } - - buf := new(bytes.Buffer) - err := jpeg.Encode(buf, resizedImage, nil) +func (e *ThumbnailEncoder) GetThumbnail(img *models.Image, maxSize int) ([]byte, error) { + reader, err := openSourceImage(img.Path) if err != nil { return nil, err } - return buf.Bytes(), nil + + buf := new(bytes.Buffer) + if _, err := buf.ReadFrom(reader); err != nil { + return nil, err + } + + _, format, err := DecodeSourceImage(img) + if err != nil { + return nil, err + } + + if format != nil && *format == "gif" { + return buf.Bytes(), nil + } + + // vips has issues loading files from stdin on Windows + if e.VipsPath != "" && runtime.GOOS != "windows" { + return e.getVipsThumbnail(buf, maxSize) + } else { + return e.getFFMPEGThumbnail(buf, format, maxSize, img.Path) + } +} + +func (e *ThumbnailEncoder) getVipsThumbnail(image *bytes.Buffer, maxSize int) ([]byte, error) { + args := []string{ + "thumbnail_source", + "[descriptor=0]", + ".jpg[Q=70,strip]", + fmt.Sprint(maxSize), + "--size", "down", + } + data, err := e.run(e.VipsPath, args, image) + + return []byte(data), err +} + +func (e *ThumbnailEncoder) getFFMPEGThumbnail(image *bytes.Buffer, format *string, maxDimensions int, path string) ([]byte, error) { + // ffmpeg spends a long sniffing image format when data is piped through stdio, so we pass the format explicitly instead + ffmpegformat := "" + if format != nil && *format == "jpeg" { + ffmpegformat = "mjpeg" + } else if format != nil && *format == "png" { + ffmpegformat = "png_pipe" + } else if format != nil && *format == "webp" { + ffmpegformat = "webp_pipe" + } else { + return nil, errors.New("unsupported image format") + } + + args := []string{ + "-f", ffmpegformat, + "-i", "-", + "-vf", fmt.Sprintf("scale=%v:%v:force_original_aspect_ratio=decrease", maxDimensions, maxDimensions), + "-c:v", "mjpeg", + "-q:v", "5", + "-f", "image2pipe", + "-", + } + data, err := e.run(e.FFMPEGPath, args, image) + + return []byte(data), err +} + +func (e *ThumbnailEncoder) run(path string, args []string, stdin *bytes.Buffer) (string, error) { + cmd := exec.Command(path, args...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + cmd.Stdin = stdin + + if err := cmd.Start(); err != nil { + return "", err + } + + err := cmd.Wait() + + if err != nil { + // error message should be in the stderr stream + logger.Errorf("image encoder error when running command <%s>: %s", strings.Join(cmd.Args, " "), stderr.String()) + return stdout.String(), err + } + + return stdout.String(), nil } diff --git a/pkg/job/progress_test.go b/pkg/job/progress_test.go index f2dbc0bc9..5bca05ae4 100644 --- a/pkg/job/progress_test.go +++ b/pkg/job/progress_test.go @@ -132,14 +132,18 @@ func TestExecuteTask(t *testing.T) { assert := assert.New(t) + m.mutex.Lock() // ensure task is added to the job details assert.Equal(taskDesciption, j.Details[0]) + m.mutex.Unlock() // allow task to finish close(c) time.Sleep(sleepTime) + m.mutex.Lock() // ensure task is removed from the job details assert.Len(j.Details, 0) + m.mutex.Unlock() } diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 8e2de2bbf..0e2a3db0b 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -41,7 +41,6 @@ func Init(logFile string, logOut bool, logLevel string) { if err != nil { fmt.Printf("Could not open '%s' for log output due to error: %s\n", logFile, err.Error()) - logFile = "" } } @@ -183,6 +182,11 @@ func Progressf(format string, args ...interface{}) { func Trace(args ...interface{}) { logger.Trace(args...) + l := &LogItem{ + Type: "trace", + Message: fmt.Sprint(args...), + } + addLogItem(l) } func Tracef(format string, args ...interface{}) { diff --git a/pkg/logger/plugin.go b/pkg/logger/plugin.go new file mode 100644 index 000000000..67e6b03e9 --- /dev/null +++ b/pkg/logger/plugin.go @@ -0,0 +1,192 @@ +package logger + +import ( + "bufio" + "fmt" + "io" + "os" + "strconv" + "strings" +) + +type PluginLogLevel struct { + char byte + Name string +} + +// Valid Level values. +var ( + TraceLevel = PluginLogLevel{ + char: 't', + Name: "trace", + } + DebugLevel = PluginLogLevel{ + char: 'd', + Name: "debug", + } + InfoLevel = PluginLogLevel{ + char: 'i', + Name: "info", + } + WarningLevel = PluginLogLevel{ + char: 'w', + Name: "warning", + } + ErrorLevel = PluginLogLevel{ + char: 'e', + Name: "error", + } + ProgressLevel = PluginLogLevel{ + char: 'p', + Name: "progress", + } + NoneLevel = PluginLogLevel{ + Name: "none", + } +) + +var validLevels = []PluginLogLevel{ + TraceLevel, + DebugLevel, + InfoLevel, + WarningLevel, + ErrorLevel, + ProgressLevel, + NoneLevel, +} + +const startLevelChar byte = 1 +const endLevelChar byte = 2 + +func (l PluginLogLevel) prefix() string { + return string([]byte{ + startLevelChar, + byte(l.char), + endLevelChar, + }) +} + +func (l PluginLogLevel) Log(args ...interface{}) { + if l.char == 0 { + return + } + + argsToUse := []interface{}{ + l.prefix(), + } + argsToUse = append(argsToUse, args...) + fmt.Fprintln(os.Stderr, argsToUse...) +} + +func (l PluginLogLevel) Logf(format string, args ...interface{}) { + if l.char == 0 { + return + } + + formatToUse := string(l.prefix()) + format + "\n" + fmt.Fprintf(os.Stderr, formatToUse, args...) +} + +// PluginLogLevelFromName returns the Level that matches the provided name or nil if +// the name does not match a valid value. +func PluginLogLevelFromName(name string) *PluginLogLevel { + for _, l := range validLevels { + if l.Name == name { + return &l + } + } + + return nil +} + +// DetectLogLevel returns the Level and the logging string for a provided line +// of plugin output. It parses the string for logging control characters and +// determines the log level, if present. If not present, the plugin output +// is returned unchanged with a nil Level. +func DetectLogLevel(line string) (*PluginLogLevel, string) { + if len(line) < 4 || line[0] != startLevelChar || line[2] != endLevelChar { + return nil, line + } + + char := line[1] + var level *PluginLogLevel + for _, l := range validLevels { + if l.char == char { + l := l // Make a copy of the loop variable + level = &l + break + } + } + + if level == nil { + return nil, line + } + + line = strings.TrimSpace(line[3:]) + + return level, line +} + +type PluginLogger struct { + Prefix string + DefaultLogLevel *PluginLogLevel + ProgressChan chan float64 +} + +func (log *PluginLogger) HandleStderrLine(line string) { + level, ll := DetectLogLevel(line) + + // if no log level, just output to info + if level == nil { + if log.DefaultLogLevel != nil { + level = log.DefaultLogLevel + } else { + level = &InfoLevel + } + } + + switch *level { + case TraceLevel: + Trace(log.Prefix, ll) + case DebugLevel: + Debug(log.Prefix, ll) + case InfoLevel: + Info(log.Prefix, ll) + case WarningLevel: + Warn(log.Prefix, ll) + case ErrorLevel: + Error(log.Prefix, ll) + case ProgressLevel: + p, err := strconv.ParseFloat(ll, 64) + if err != nil { + Errorf("Error parsing progress value '%s': %s", ll, err.Error()) + } else { + // only pass progress through if channel present + if log.ProgressChan != nil { + // don't block on this + select { + case log.ProgressChan <- p: + default: + } + } + } + } +} + +func (log *PluginLogger) HandlePluginStdErr(pluginStdErr io.ReadCloser) { + // pipe plugin stderr to our logging + scanner := bufio.NewScanner(pluginStdErr) + for scanner.Scan() { + str := scanner.Text() + if str != "" { + log.HandleStderrLine(str) + } + } + + str := scanner.Text() + if str != "" { + log.HandleStderrLine(str) + } + + pluginStdErr.Close() +} diff --git a/pkg/manager/apikey.go b/pkg/manager/apikey.go index a01a9b221..09147c9db 100644 --- a/pkg/manager/apikey.go +++ b/pkg/manager/apikey.go @@ -4,7 +4,7 @@ import ( "errors" "time" - "github.com/dgrijalva/jwt-go" + "github.com/golang-jwt/jwt/v4" "github.com/stashapp/stash/pkg/manager/config" ) diff --git a/pkg/manager/config/config.go b/pkg/manager/config/config.go index c412880ca..92235394b 100644 --- a/pkg/manager/config/config.go +++ b/pkg/manager/config/config.go @@ -3,7 +3,7 @@ package config import ( "errors" "fmt" - "io/ioutil" + "os" "path/filepath" "regexp" "runtime" @@ -16,6 +16,7 @@ import ( "github.com/spf13/viper" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/paths" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" @@ -83,6 +84,9 @@ const previewExcludeStartDefault = "0" const PreviewExcludeEnd = "preview_exclude_end" const previewExcludeEndDefault = "0" +const WriteImageThumbnails = "write_image_thumbnails" +const writeImageThumbnailsDefault = true + const Host = "host" const Port = "port" const ExternalHost = "external_host" @@ -134,6 +138,13 @@ const SlideshowDelay = "slideshow_delay" const HandyKey = "handy_key" const FunscriptOffset = "funscript_offset" +// Security +const TrustedProxies = "trusted_proxies" +const dangerousAllowPublicWithoutAuth = "dangerous_allow_public_without_auth" +const dangerousAllowPublicWithoutAuthDefault = "false" +const SecurityTripwireAccessedFromPublicInternet = "security_tripwire_accessed_from_public_internet" +const securityTripwireAccessedFromPublicInternetDefault = "" + // DLNA options const DLNAServerName = "dlna.server_name" const DLNADefaultEnabled = "dlna.default_enabled" @@ -453,7 +464,10 @@ func (i *Instance) GetStashBoxes() []*models.StashBox { i.RLock() defer i.RUnlock() var boxes []*models.StashBox - viper.UnmarshalKey(StashBoxes, &boxes) + if err := viper.UnmarshalKey(StashBoxes, &boxes); err != nil { + logger.Warnf("error in unmarshalkey: %v", err) + } + return boxes } @@ -591,6 +605,14 @@ func (i *Instance) GetMaxStreamingTranscodeSize() models.StreamingResolutionEnum return models.StreamingResolutionEnum(ret) } +// IsWriteImageThumbnails returns true if image thumbnails should be written +// to disk after generating on the fly. +func (i *Instance) IsWriteImageThumbnails() bool { + i.RLock() + defer i.RUnlock() + return viper.GetBool(WriteImageThumbnails) +} + func (i *Instance) GetAPIKey() string { i.RLock() defer i.RUnlock() @@ -656,17 +678,21 @@ func (i *Instance) ValidateStashBoxes(boxes []*models.StashBoxInput) error { re, err := regexp.Compile("^http.*graphql$") if err != nil { - return errors.New("Failure to generate regular expression") + return errors.New("failure to generate regular expression") } for _, box := range boxes { if box.APIKey == "" { + //lint:ignore ST1005 Stash-box is a name return errors.New("Stash-box API Key cannot be blank") } else if box.Endpoint == "" { + //lint:ignore ST1005 Stash-box is a name return errors.New("Stash-box Endpoint cannot be blank") } else if !re.Match([]byte(box.Endpoint)) { + //lint:ignore ST1005 Stash-box is a name return errors.New("Stash-box Endpoint is invalid") } else if isMulti && box.Name == "" { + //lint:ignore ST1005 Stash-box is a name return errors.New("Stash-box Name cannot be blank") } } @@ -781,7 +807,7 @@ func (i *Instance) GetCSS() string { return "" } - buf, err := ioutil.ReadFile(fn) + buf, err := os.ReadFile(fn) if err != nil { return "" @@ -797,7 +823,9 @@ func (i *Instance) SetCSS(css string) { buf := []byte(css) - ioutil.WriteFile(fn, buf, 0777) + if err := os.WriteFile(fn, buf, 0777); err != nil { + logger.Warnf("error while writing %v bytes to %v: %v", len(buf), fn, err) + } } func (i *Instance) GetCSSEnabled() bool { @@ -817,6 +845,31 @@ func (i *Instance) GetFunscriptOffset() int { return viper.GetInt(FunscriptOffset) } +// GetTrustedProxies returns a comma separated list of ip addresses that should allow proxying. +// When empty, allow from any private network +func (i *Instance) GetTrustedProxies() []string { + i.RLock() + defer i.RUnlock() + return viper.GetStringSlice(TrustedProxies) +} + +// GetDangerousAllowPublicWithoutAuth determines if the security feature is enabled. +// See https://github.com/stashapp/stash/wiki/Authentication-Required-When-Accessing-Stash-From-the-Internet +func (i *Instance) GetDangerousAllowPublicWithoutAuth() bool { + i.RLock() + defer i.RUnlock() + return viper.GetBool(dangerousAllowPublicWithoutAuth) +} + +// GetSecurityTripwireAccessedFromPublicInternet returns a public IP address if stash +// has been accessed from the public internet, with no auth enabled, and +// DangerousAllowPublicWithoutAuth disabled. Returns an empty string otherwise. +func (i *Instance) GetSecurityTripwireAccessedFromPublicInternet() string { + i.RLock() + defer i.RUnlock() + return viper.GetString(SecurityTripwireAccessedFromPublicInternet) +} + // GetDLNAServerName returns the visible name of the DLNA server. If empty, // "stash" will be used. func (i *Instance) GetDLNAServerName() string { @@ -909,6 +962,14 @@ func (i *Instance) GetMaxUploadSize() int64 { return ret << 20 } +// ActivatePublicAccessTripwire sets the security_tripwire_accessed_from_public_internet +// config field to the provided IP address to indicate that stash has been accessed +// from this public IP without authentication. +func (i *Instance) ActivatePublicAccessTripwire(requestIP string) error { + i.Set(SecurityTripwireAccessedFromPublicInternet, requestIP) + return i.Write() +} + func (i *Instance) Validate() error { i.RLock() defer i.RUnlock() @@ -941,8 +1002,7 @@ func (i *Instance) SetChecksumDefaultValues(defaultAlgorithm models.HashAlgorith viper.SetDefault(CalculateMD5, usingMD5) } -func (i *Instance) setDefaultValues() error { - +func (i *Instance) setDefaultValues(write bool) error { // read data before write lock scope defaultDatabaseFilePath := i.GetDefaultDatabaseFilePath() defaultScrapersPath := i.GetDefaultScrapersPath() @@ -958,19 +1018,37 @@ func (i *Instance) setDefaultValues() error { viper.SetDefault(PreviewAudio, previewAudioDefault) viper.SetDefault(SoundOnPreview, false) + viper.SetDefault(WriteImageThumbnails, writeImageThumbnailsDefault) + viper.SetDefault(Database, defaultDatabaseFilePath) + viper.SetDefault(dangerousAllowPublicWithoutAuth, dangerousAllowPublicWithoutAuthDefault) + viper.SetDefault(SecurityTripwireAccessedFromPublicInternet, securityTripwireAccessedFromPublicInternetDefault) + // Set generated to the metadata path for backwards compat viper.SetDefault(Generated, viper.GetString(Metadata)) // Set default scrapers and plugins paths viper.SetDefault(ScrapersPath, defaultScrapersPath) viper.SetDefault(PluginsPath, defaultPluginsPath) - return viper.WriteConfig() + if write { + return viper.WriteConfig() + } + + return nil } // SetInitialConfig fills in missing required config fields func (i *Instance) SetInitialConfig() error { + return i.setInitialConfig(true) +} + +// SetInitialMemoryConfig fills in missing required config fields without writing the configuration +func (i *Instance) SetInitialMemoryConfig() error { + return i.setInitialConfig(false) +} + +func (i *Instance) setInitialConfig(write bool) error { // generate some api keys const apiKeyLength = 32 @@ -984,7 +1062,7 @@ func (i *Instance) SetInitialConfig() error { i.Set(SessionStoreKey, sessionStoreKey) } - return i.setDefaultValues() + return i.setDefaultValues(write) } func (i *Instance) FinalizeSetup() { diff --git a/pkg/manager/config/config_concurrency_test.go b/pkg/manager/config/config_concurrency_test.go index 2f86f332e..cc6b56d67 100644 --- a/pkg/manager/config/config_concurrency_test.go +++ b/pkg/manager/config/config_concurrency_test.go @@ -13,11 +13,13 @@ func TestConcurrentConfigAccess(t *testing.T) { //const loops = 1000 const loops = 200 var wg sync.WaitGroup - for t := 0; t < workers; t++ { + for k := 0; k < workers; k++ { wg.Add(1) - go func() { + go func(wk int) { for l := 0; l < loops; l++ { - i.SetInitialConfig() + if err := i.SetInitialMemoryConfig(); err != nil { + t.Errorf("Failure setting initial configuration in worker %v iteration %v: %v", wk, l, err) + } i.HasCredentials() i.GetCPUProfilePath() @@ -93,7 +95,7 @@ func TestConcurrentConfigAccess(t *testing.T) { i.Set(MaxUploadSize, i.GetMaxUploadSize()) } wg.Done() - }() + }(k) } wg.Wait() diff --git a/pkg/manager/config/init.go b/pkg/manager/config/init.go index 49f168b66..7117a18af 100644 --- a/pkg/manager/config/init.go +++ b/pkg/manager/config/init.go @@ -112,16 +112,22 @@ func initFlags() flagStruct { } func initEnvs() { - viper.SetEnvPrefix("stash") // will be uppercased automatically - viper.BindEnv("host") // STASH_HOST - viper.BindEnv("port") // STASH_PORT - viper.BindEnv("external_host") // STASH_EXTERNAL_HOST - viper.BindEnv("generated") // STASH_GENERATED - viper.BindEnv("metadata") // STASH_METADATA - viper.BindEnv("cache") // STASH_CACHE + viper.SetEnvPrefix("stash") // will be uppercased automatically + bindEnv("host") // STASH_HOST + bindEnv("port") // STASH_PORT + bindEnv("external_host") // STASH_EXTERNAL_HOST + bindEnv("generated") // STASH_GENERATED + bindEnv("metadata") // STASH_METADATA + bindEnv("cache") // STASH_CACHE // only set stash config flag if not already set if instance.GetStashPaths() == nil { - viper.BindEnv("stash") // STASH_STASH + bindEnv("stash") // STASH_STASH + } +} + +func bindEnv(key string) { + if err := viper.BindEnv(key); err != nil { + panic(fmt.Sprintf("unable to set environment key (%v): %v", key, err)) } } diff --git a/pkg/manager/filename_parser.go b/pkg/manager/filename_parser.go index 991261941..9dbfecd81 100644 --- a/pkg/manager/filename_parser.go +++ b/pkg/manager/filename_parser.go @@ -3,6 +3,7 @@ package manager import ( "database/sql" "errors" + "github.com/stashapp/stash/pkg/studio" "path/filepath" "regexp" "strconv" @@ -11,8 +12,6 @@ import ( "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/tag" - - "github.com/jmoiron/sqlx" ) type parserField struct { @@ -52,10 +51,6 @@ func (f parserField) replaceInPattern(pattern string) string { return string(f.fieldRegex.ReplaceAllString(pattern, f.regex)) } -func (f parserField) getFieldPattern() string { - return "{" + f.field + "}" -} - var validFields map[string]parserField var escapeCharRE *regexp.Regexp var capitalizeTitleRE *regexp.Regexp @@ -405,26 +400,6 @@ func (m parseMapper) parse(scene *models.Scene) *sceneHolder { return sceneHolder } -type performerQueryer interface { - FindByNames(names []string, tx *sqlx.Tx, nocase bool) ([]*models.Performer, error) -} - -type sceneQueryer interface { - QueryByPathRegex(findFilter *models.FindFilterType) ([]*models.Scene, int) -} - -type tagQueryer interface { - FindByName(name string, tx *sqlx.Tx, nocase bool) (*models.Tag, error) -} - -type studioQueryer interface { - FindByName(name string, tx *sqlx.Tx, nocase bool) (*models.Studio, error) -} - -type movieQueryer interface { - FindByName(name string, tx *sqlx.Tx, nocase bool) (*models.Movie, error) -} - type SceneFilenameParser struct { Pattern string ParserInput models.SceneParserInput @@ -563,7 +538,12 @@ func (p *SceneFilenameParser) queryStudio(qb models.StudioReader, studioName str return ret } - ret, _ := qb.FindByName(studioName, true) + ret, _ := studio.ByName(qb, studioName) + + // try to match on alias + if ret == nil { + ret, _ = studio.ByAlias(qb, studioName) + } // add result to cache p.studioCache[studioName] = ret diff --git a/pkg/manager/generator_sprite.go b/pkg/manager/generator_sprite.go index bbd1b92c4..2aebb827e 100644 --- a/pkg/manager/generator_sprite.go +++ b/pkg/manager/generator_sprite.go @@ -4,7 +4,6 @@ import ( "fmt" "image" "image/color" - "io/ioutil" "math" "os" "path/filepath" @@ -141,7 +140,7 @@ func (g *SpriteGenerator) generateSpriteVTT(encoder *ffmpeg.Encoder) error { } vtt := strings.Join(vttLines, "\n") - return ioutil.WriteFile(g.VTTOutputPath, []byte(vtt), 0644) + return os.WriteFile(g.VTTOutputPath, []byte(vtt), 0644) } func (g *SpriteGenerator) imageExists() bool { diff --git a/pkg/manager/image.go b/pkg/manager/image.go index 8af987233..b0962f554 100644 --- a/pkg/manager/image.go +++ b/pkg/manager/image.go @@ -61,10 +61,13 @@ func walkGalleryZip(path string, walkFunc func(file *zip.File) error) error { func countImagesInZip(path string) int { ret := 0 - walkGalleryZip(path, func(file *zip.File) error { + err := walkGalleryZip(path, func(file *zip.File) error { ret++ return nil }) + if err != nil { + logger.Warnf("Error while walking gallery zip: %v", err) + } return ret } diff --git a/pkg/manager/jsonschema/studio.go b/pkg/manager/jsonschema/studio.go index ed1b6d144..ee793acbc 100644 --- a/pkg/manager/jsonschema/studio.go +++ b/pkg/manager/jsonschema/studio.go @@ -17,6 +17,7 @@ type Studio struct { UpdatedAt models.JSONTime `json:"updated_at,omitempty"` Rating int `json:"rating,omitempty"` Details string `json:"details,omitempty"` + Aliases []string `json:"aliases,omitempty"` } func LoadStudioFile(filePath string) (*Studio, error) { diff --git a/pkg/manager/jsonschema/tag.go b/pkg/manager/jsonschema/tag.go index 66fa910ab..fbb85a835 100644 --- a/pkg/manager/jsonschema/tag.go +++ b/pkg/manager/jsonschema/tag.go @@ -12,6 +12,7 @@ type Tag struct { Name string `json:"name,omitempty"` Aliases []string `json:"aliases,omitempty"` Image string `json:"image,omitempty"` + Parents []string `json:"parents,omitempty"` CreatedAt models.JSONTime `json:"created_at,omitempty"` UpdatedAt models.JSONTime `json:"updated_at,omitempty"` } diff --git a/pkg/manager/jsonschema/utils.go b/pkg/manager/jsonschema/utils.go index 2844026f7..50f950866 100644 --- a/pkg/manager/jsonschema/utils.go +++ b/pkg/manager/jsonschema/utils.go @@ -2,15 +2,11 @@ package jsonschema import ( "bytes" - - "io/ioutil" - "time" + "os" jsoniter "github.com/json-iterator/go" ) -var nilTime = (time.Time{}).UnixNano() - func CompareJSON(a interface{}, b interface{}) bool { aBuf, _ := encode(a) bBuf, _ := encode(b) @@ -22,7 +18,7 @@ func marshalToFile(filePath string, j interface{}) error { if err != nil { return err } - return ioutil.WriteFile(filePath, data, 0644) + return os.WriteFile(filePath, data, 0644) } func encode(j interface{}) ([]byte, error) { diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 638332619..03f0fe260 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -97,6 +97,8 @@ func Initialize() *singleton { panic(err) } } + + initSecurity(cfg) } else { cfgFile := cfg.GetConfigFile() if cfgFile != "" { @@ -110,17 +112,27 @@ func Initialize() *singleton { logger.Warnf("config file %snot found. Assuming new system...", cfgFile) } - initFFMPEG() + if err = initFFMPEG(); err != nil { + logger.Warnf("could not initialize FFMPEG subsystem: %v", err) + } // if DLNA is enabled, start it now if instance.Config.GetDLNADefaultEnabled() { - instance.DLNAService.Start(nil) + if err := instance.DLNAService.Start(nil); err != nil { + logger.Warnf("could not start DLNA service: %v", err) + } } }) return instance } +func initSecurity(cfg *config.Instance) { + if err := session.CheckExternalAccessTripwire(cfg); err != nil { + session.LogExternalAccessError(*err) + } +} + func initProfiling(cpuProfilePath string) { if cpuProfilePath == "" { return @@ -134,7 +146,9 @@ func initProfiling(cpuProfilePath string) { logger.Infof("profiling to %s", cpuProfilePath) // StopCPUProfile is defer called in main - pprof.StartCPUProfile(f) + if err = pprof.StartCPUProfile(f); err != nil { + logger.Warnf("could not start CPU profiling: %v", err) + } } func initFFMPEG() error { @@ -182,7 +196,9 @@ func initLog() { // configuration has been set. Should only be called if the configuration // is valid. func (s *singleton) PostInit() error { - s.Config.SetInitialConfig() + if err := s.Config.SetInitialConfig(); err != nil { + logger.Warnf("could not set initial configuration: %v", err) + } s.Paths = paths.NewPaths(s.Config.GetGeneratedPath()) s.RefreshConfig() @@ -201,8 +217,12 @@ func (s *singleton) PostInit() error { const deleteTimeout = 1 * time.Second utils.Timeout(func() { - utils.EmptyDir(instance.Paths.Generated.Downloads) - utils.EmptyDir(instance.Paths.Generated.Tmp) + if err := utils.EmptyDir(instance.Paths.Generated.Downloads); err != nil { + logger.Warnf("could not empty Downloads directory: %v", err) + } + if err := utils.EmptyDir(instance.Paths.Generated.Tmp); err != nil { + logger.Warnf("could not empty Tmp directory: %v", err) + } }, deleteTimeout, func(done chan struct{}) { logger.Info("Please wait. Deleting temporary files...") // print <-done // and wait for deletion @@ -236,11 +256,21 @@ func (s *singleton) RefreshConfig() { s.Paths = paths.NewPaths(s.Config.GetGeneratedPath()) config := s.Config if config.Validate() == nil { - utils.EnsureDir(s.Paths.Generated.Screenshots) - utils.EnsureDir(s.Paths.Generated.Vtt) - utils.EnsureDir(s.Paths.Generated.Markers) - utils.EnsureDir(s.Paths.Generated.Transcodes) - utils.EnsureDir(s.Paths.Generated.Downloads) + if err := utils.EnsureDir(s.Paths.Generated.Screenshots); err != nil { + logger.Warnf("could not create directory for Screenshots: %v", err) + } + if err := utils.EnsureDir(s.Paths.Generated.Vtt); err != nil { + logger.Warnf("could not create directory for VTT: %v", err) + } + if err := utils.EnsureDir(s.Paths.Generated.Markers); err != nil { + logger.Warnf("could not create directory for Markers: %v", err) + } + if err := utils.EnsureDir(s.Paths.Generated.Transcodes); err != nil { + logger.Warnf("could not create directory for Transcodes: %v", err) + } + if err := utils.EnsureDir(s.Paths.Generated.Downloads); err != nil { + logger.Warnf("could not create directory for Downloads: %v", err) + } } } @@ -304,7 +334,9 @@ func (s *singleton) Setup(input models.SetupInput) error { s.Config.FinalizeSetup() - initFFMPEG() + if err := initFFMPEG(); err != nil { + return fmt.Errorf("error initializing FFMPEG subsystem: %v", err) + } return nil } @@ -378,3 +410,10 @@ func (s *singleton) GetSystemStatus() *models.SystemStatus { ConfigPath: &configFile, } } + +// Shutdown gracefully stops the manager +func (s *singleton) Shutdown() error { + // TODO: Each part of the manager needs to gracefully stop at some point + // for now, we just close the database. + return database.Close() +} diff --git a/pkg/manager/manager_tasks.go b/pkg/manager/manager_tasks.go index 6173c30ed..a626dffad 100644 --- a/pkg/manager/manager_tasks.go +++ b/pkg/manager/manager_tasks.go @@ -82,9 +82,6 @@ func (s *singleton) Import(ctx context.Context) (int, error) { } j := job.MakeJobExec(func(ctx context.Context, progress *job.Progress) { - var wg sync.WaitGroup - wg.Add(1) - task := ImportTask{ txnManager: s.TxnManager, BaseDir: metadataPath, @@ -93,7 +90,7 @@ func (s *singleton) Import(ctx context.Context) (int, error) { MissingRefBehaviour: models.ImportMissingRefEnumFail, fileNamingAlgorithm: config.GetVideoFileNamingAlgorithm(), } - task.Start(&wg) + task.Start() }) return s.JobManager.Add(ctx, "Importing...", j), nil @@ -125,7 +122,8 @@ func (s *singleton) RunSingleTask(ctx context.Context, t Task) int { wg.Add(1) j := job.MakeJobExec(func(ctx context.Context, progress *job.Progress) { - t.Start(&wg) + t.Start() + wg.Done() }) return s.JobManager.Add(ctx, t.GetDescription(), j) @@ -163,7 +161,9 @@ func (s *singleton) Generate(ctx context.Context, input models.GenerateMetadataI if err := s.validateFFMPEG(); err != nil { return 0, err } - instance.Paths.Generated.EnsureTmpDir() + if err := instance.Paths.Generated.EnsureTmpDir(); err != nil { + logger.Warnf("could not generate temporary directory: %v", err) + } sceneIDs, err := utils.StringSliceToIntSlice(input.SceneIDs) if err != nil { @@ -250,14 +250,18 @@ func (s *singleton) Generate(ctx context.Context, input models.GenerateMetadataI // Start measuring how long the generate has taken. (consider moving this up) start := time.Now() - instance.Paths.Generated.EnsureTmpDir() + if err = instance.Paths.Generated.EnsureTmpDir(); err != nil { + logger.Warnf("could not create temporary directory: %v", err) + } for _, scene := range scenes { progress.Increment() if job.IsCancelled(ctx) { logger.Info("Stopping due to user request") wg.Wait() - instance.Paths.Generated.EmptyTmpDir() + if err := instance.Paths.Generated.EmptyTmpDir(); err != nil { + logger.Warnf("failure emptying temporary directory: %v", err) + } return } @@ -266,7 +270,7 @@ func (s *singleton) Generate(ctx context.Context, input models.GenerateMetadataI continue } - if input.Sprites { + if utils.IsTrue(input.Sprites) { task := GenerateSpriteTask{ Scene: *scene, Overwrite: overwrite, @@ -274,38 +278,43 @@ func (s *singleton) Generate(ctx context.Context, input models.GenerateMetadataI } wg.Add() go progress.ExecuteTask(fmt.Sprintf("Generating sprites for %s", scene.Path), func() { - task.Start(&wg) + task.Start() + wg.Done() }) } - if input.Previews { + if utils.IsTrue(input.Previews) { task := GeneratePreviewTask{ Scene: *scene, - ImagePreview: input.ImagePreviews, + ImagePreview: utils.IsTrue(input.ImagePreviews), Options: *generatePreviewOptions, Overwrite: overwrite, fileNamingAlgorithm: fileNamingAlgo, } wg.Add() go progress.ExecuteTask(fmt.Sprintf("Generating preview for %s", scene.Path), func() { - task.Start(&wg) + task.Start() + wg.Done() }) } - if input.Markers { + if utils.IsTrue(input.Markers) { wg.Add() task := GenerateMarkersTask{ TxnManager: s.TxnManager, Scene: scene, Overwrite: overwrite, fileNamingAlgorithm: fileNamingAlgo, + ImagePreview: utils.IsTrue(input.MarkerImagePreviews), + Screenshot: utils.IsTrue(input.MarkerScreenshots), } go progress.ExecuteTask(fmt.Sprintf("Generating markers for %s", scene.Path), func() { - task.Start(&wg) + task.Start() + wg.Done() }) } - if input.Transcodes { + if utils.IsTrue(input.Transcodes) { wg.Add() task := GenerateTranscodeTask{ Scene: *scene, @@ -313,11 +322,12 @@ func (s *singleton) Generate(ctx context.Context, input models.GenerateMetadataI fileNamingAlgorithm: fileNamingAlgo, } go progress.ExecuteTask(fmt.Sprintf("Generating transcode for %s", scene.Path), func() { - task.Start(&wg) + task.Start() + wg.Done() }) } - if input.Phashes { + if utils.IsTrue(input.Phashes) { task := GeneratePhashTask{ Scene: *scene, fileNamingAlgorithm: fileNamingAlgo, @@ -326,7 +336,8 @@ func (s *singleton) Generate(ctx context.Context, input models.GenerateMetadataI } wg.Add() go progress.ExecuteTask(fmt.Sprintf("Generating phash for %s", scene.Path), func() { - task.Start(&wg) + task.Start() + wg.Done() }) } } @@ -338,7 +349,9 @@ func (s *singleton) Generate(ctx context.Context, input models.GenerateMetadataI if job.IsCancelled(ctx) { logger.Info("Stopping due to user request") wg.Wait() - instance.Paths.Generated.EmptyTmpDir() + if err := instance.Paths.Generated.EmptyTmpDir(); err != nil { + logger.Warnf("failure emptying temporary directory: %v", err) + } elapsed := time.Since(start) logger.Info(fmt.Sprintf("Generate finished (%s)", elapsed)) return @@ -357,13 +370,16 @@ func (s *singleton) Generate(ctx context.Context, input models.GenerateMetadataI fileNamingAlgorithm: fileNamingAlgo, } go progress.ExecuteTask(fmt.Sprintf("Generating marker preview for marker ID %d", marker.ID), func() { - task.Start(&wg) + task.Start() + wg.Done() }) } wg.Wait() - instance.Paths.Generated.EmptyTmpDir() + if err = instance.Paths.Generated.EmptyTmpDir(); err != nil { + logger.Warnf("failure emptying temporary directory: %v", err) + } elapsed := time.Since(start) logger.Info(fmt.Sprintf("Generate finished (%s)", elapsed)) }) @@ -381,7 +397,9 @@ func (s *singleton) GenerateScreenshot(ctx context.Context, sceneId string, at f // generate default screenshot if at is nil func (s *singleton) generateScreenshot(ctx context.Context, sceneId string, at *float64) int { - instance.Paths.Generated.EnsureTmpDir() + if err := instance.Paths.Generated.EnsureTmpDir(); err != nil { + logger.Warnf("failure generating screenshot: %v", err) + } j := job.MakeJobExec(func(ctx context.Context, progress *job.Progress) { sceneIdInt, err := strconv.Atoi(sceneId) @@ -407,9 +425,7 @@ func (s *singleton) generateScreenshot(ctx context.Context, sceneId string, at * fileNamingAlgorithm: config.GetInstance().GetVideoFileNamingAlgorithm(), } - var wg sync.WaitGroup - wg.Add(1) - task.Start(&wg) + task.Start() logger.Infof("Generate screenshot finished") }) @@ -593,7 +609,11 @@ func (s *singleton) MigrateHash(ctx context.Context) int { wg.Add(1) task := MigrateHashTask{Scene: scene, fileNamingAlgorithm: fileNamingAlgo} - go task.Start(&wg) + go func() { + task.Start() + wg.Done() + }() + wg.Wait() } @@ -635,7 +655,7 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, input models.Generate logger.Infof("Counting content to generate...") for _, scene := range scenes { if scene != nil { - if input.Sprites { + if utils.IsTrue(input.Sprites) { task := GenerateSpriteTask{ Scene: *scene, fileNamingAlgorithm: fileNamingAlgo, @@ -646,10 +666,10 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, input models.Generate } } - if input.Previews { + if utils.IsTrue(input.Previews) { task := GeneratePreviewTask{ Scene: *scene, - ImagePreview: input.ImagePreviews, + ImagePreview: utils.IsTrue(input.ImagePreviews), fileNamingAlgorithm: fileNamingAlgo, } @@ -658,12 +678,12 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, input models.Generate totals.previews++ } - if input.ImagePreviews && (overwrite || !task.doesImagePreviewExist(sceneHash)) { + if utils.IsTrue(input.ImagePreviews) && (overwrite || !task.doesImagePreviewExist(sceneHash)) { totals.imagePreviews++ } } - if input.Markers { + if utils.IsTrue(input.Markers) { task := GenerateMarkersTask{ TxnManager: s.TxnManager, Scene: scene, @@ -673,7 +693,7 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, input models.Generate totals.markers += int64(task.isMarkerNeeded()) } - if input.Transcodes { + if utils.IsTrue(input.Transcodes) { task := GenerateTranscodeTask{ Scene: *scene, Overwrite: overwrite, @@ -684,7 +704,7 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, input models.Generate } } - if input.Phashes { + if utils.IsTrue(input.Phashes) { task := GeneratePhashTask{ Scene: *scene, fileNamingAlgorithm: fileNamingAlgo, @@ -797,7 +817,8 @@ func (s *singleton) StashBoxBatchPerformerTag(ctx context.Context, input models. for _, task := range tasks { wg.Add(1) progress.ExecuteTask(task.Description(), func() { - task.Start(&wg) + task.Start() + wg.Done() }) progress.Increment() diff --git a/pkg/manager/paths/paths_generated.go b/pkg/manager/paths/paths_generated.go index 234f3918b..299ddd56b 100644 --- a/pkg/manager/paths/paths_generated.go +++ b/pkg/manager/paths/paths_generated.go @@ -2,9 +2,10 @@ package paths import ( "fmt" - "io/ioutil" + "os" "path/filepath" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/utils" ) @@ -37,26 +38,30 @@ func (gp *generatedPaths) GetTmpPath(fileName string) string { return filepath.Join(gp.Tmp, fileName) } -func (gp *generatedPaths) EnsureTmpDir() { - utils.EnsureDir(gp.Tmp) +func (gp *generatedPaths) EnsureTmpDir() error { + return utils.EnsureDir(gp.Tmp) } -func (gp *generatedPaths) EmptyTmpDir() { - utils.EmptyDir(gp.Tmp) +func (gp *generatedPaths) EmptyTmpDir() error { + return utils.EmptyDir(gp.Tmp) } -func (gp *generatedPaths) RemoveTmpDir() { - utils.RemoveDir(gp.Tmp) +func (gp *generatedPaths) RemoveTmpDir() error { + return utils.RemoveDir(gp.Tmp) } func (gp *generatedPaths) TempDir(pattern string) (string, error) { - gp.EnsureTmpDir() - ret, err := ioutil.TempDir(gp.Tmp, pattern) + if err := gp.EnsureTmpDir(); err != nil { + logger.Warnf("Could not ensure existence of a temporary directory: %v", err) + } + ret, err := os.MkdirTemp(gp.Tmp, pattern) if err != nil { return "", err } - utils.EmptyDir(ret) + if err = utils.EmptyDir(ret); err != nil { + logger.Warnf("could not recursively empty dir: %v", err) + } return ret, nil } diff --git a/pkg/manager/paths/paths_json.go b/pkg/manager/paths/paths_json.go index c7f3b7490..1bd6a5e59 100644 --- a/pkg/manager/paths/paths_json.go +++ b/pkg/manager/paths/paths_json.go @@ -3,6 +3,7 @@ package paths import ( "path/filepath" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/utils" ) @@ -43,14 +44,30 @@ func GetJSONPaths(baseDir string) *JSONPaths { func EnsureJSONDirs(baseDir string) { jsonPaths := GetJSONPaths(baseDir) - utils.EnsureDir(jsonPaths.Metadata) - utils.EnsureDir(jsonPaths.Scenes) - utils.EnsureDir(jsonPaths.Images) - utils.EnsureDir(jsonPaths.Galleries) - utils.EnsureDir(jsonPaths.Performers) - utils.EnsureDir(jsonPaths.Studios) - utils.EnsureDir(jsonPaths.Movies) - utils.EnsureDir(jsonPaths.Tags) + if err := utils.EnsureDir(jsonPaths.Metadata); err != nil { + logger.Warnf("couldn't create directories for Metadata: %v", err) + } + if err := utils.EnsureDir(jsonPaths.Scenes); err != nil { + logger.Warnf("couldn't create directories for Scenes: %v", err) + } + if err := utils.EnsureDir(jsonPaths.Images); err != nil { + logger.Warnf("couldn't create directories for Images: %v", err) + } + if err := utils.EnsureDir(jsonPaths.Galleries); err != nil { + logger.Warnf("couldn't create directories for Galleries: %v", err) + } + if err := utils.EnsureDir(jsonPaths.Performers); err != nil { + logger.Warnf("couldn't create directories for Performers: %v", err) + } + if err := utils.EnsureDir(jsonPaths.Studios); err != nil { + logger.Warnf("couldn't create directories for Studios: %v", err) + } + if err := utils.EnsureDir(jsonPaths.Movies); err != nil { + logger.Warnf("couldn't create directories for Movies: %v", err) + } + if err := utils.EnsureDir(jsonPaths.Tags); err != nil { + logger.Warnf("couldn't create directories for Tags: %v", err) + } } func (jp *JSONPaths) PerformerJSONPath(checksum string) string { diff --git a/pkg/manager/paths/paths_scene_markers.go b/pkg/manager/paths/paths_scene_markers.go index e792c649f..3d9dbd6a6 100644 --- a/pkg/manager/paths/paths_scene_markers.go +++ b/pkg/manager/paths/paths_scene_markers.go @@ -22,3 +22,7 @@ func (sp *sceneMarkerPaths) GetStreamPath(checksum string, seconds int) string { func (sp *sceneMarkerPaths) GetStreamPreviewImagePath(checksum string, seconds int) string { return filepath.Join(sp.generated.Markers, checksum, strconv.Itoa(seconds)+".webp") } + +func (sp *sceneMarkerPaths) GetStreamScreenshotPath(checksum string, seconds int) string { + return filepath.Join(sp.generated.Markers, checksum, strconv.Itoa(seconds)+".jpg") +} diff --git a/pkg/manager/running_streams.go b/pkg/manager/running_streams.go index 0b3357277..99446e614 100644 --- a/pkg/manager/running_streams.go +++ b/pkg/manager/running_streams.go @@ -91,10 +91,16 @@ func (s *SceneServer) ServeScreenshot(scene *models.Scene, w http.ResponseWriter http.ServeFile(w, r, filepath) } else { var cover []byte - s.TXNManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { + err := s.TXNManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { cover, _ = repo.Scene().GetCover(scene.ID) return nil }) - utils.ServeImage(cover, w, r) + if err != nil { + logger.Warnf("read transaction failed while serving screenshot: %v", err) + } + + if err = utils.ServeImage(cover, w, r); err != nil { + logger.Warnf("unable to serve screenshot image: %v", err) + } } } diff --git a/pkg/manager/scene.go b/pkg/manager/scene.go index e262dda18..58ce2e65d 100644 --- a/pkg/manager/scene.go +++ b/pkg/manager/scene.go @@ -148,6 +148,7 @@ func DeleteGeneratedSceneFiles(scene *models.Scene, fileNamingAlgo models.HashAl func DeleteSceneMarkerFiles(scene *models.Scene, seconds int, fileNamingAlgo models.HashAlgorithm) { videoPath := GetInstance().Paths.SceneMarkers.GetStreamPath(scene.GetHash(fileNamingAlgo), seconds) imagePath := GetInstance().Paths.SceneMarkers.GetStreamPreviewImagePath(scene.GetHash(fileNamingAlgo), seconds) + screenshotPath := GetInstance().Paths.SceneMarkers.GetStreamScreenshotPath(scene.GetHash(fileNamingAlgo), seconds) exists, _ := utils.FileExists(videoPath) if exists { @@ -161,7 +162,15 @@ func DeleteSceneMarkerFiles(scene *models.Scene, seconds int, fileNamingAlgo mod if exists { err := os.Remove(imagePath) if err != nil { - logger.Warnf("Could not delete file %s: %s", videoPath, err.Error()) + logger.Warnf("Could not delete file %s: %s", imagePath, err.Error()) + } + } + + exists, _ = utils.FileExists(screenshotPath) + if exists { + err := os.Remove(screenshotPath) + if err != nil { + logger.Warnf("Could not delete file %s: %s", screenshotPath, err.Error()) } } } diff --git a/pkg/manager/screenshot.go b/pkg/manager/screenshot.go index fc417ede7..739162491 100644 --- a/pkg/manager/screenshot.go +++ b/pkg/manager/screenshot.go @@ -2,6 +2,7 @@ package manager import ( "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/logger" ) func makeScreenshot(probeResult ffmpeg.VideoFile, outputPath string, quality int, width int, time float64) { @@ -12,5 +13,8 @@ func makeScreenshot(probeResult ffmpeg.VideoFile, outputPath string, quality int Time: time, Width: width, } - encoder.Screenshot(probeResult, options) + + if err := encoder.Screenshot(probeResult, options); err != nil { + logger.Warnf("[encoder] failure to generate screenshot: %v", err) + } } diff --git a/pkg/manager/task.go b/pkg/manager/task.go index c8ab4af67..9906948c8 100644 --- a/pkg/manager/task.go +++ b/pkg/manager/task.go @@ -1,8 +1,6 @@ package manager -import "sync" - type Task interface { - Start(wg *sync.WaitGroup) + Start() GetDescription() string } diff --git a/pkg/manager/task_autotag.go b/pkg/manager/task_autotag.go index 6f577ffd3..60b86771b 100644 --- a/pkg/manager/task_autotag.go +++ b/pkg/manager/task_autotag.go @@ -215,13 +215,18 @@ func (j *autoTagJob) autoTagStudios(ctx context.Context, progress *job.Progress, } if err := j.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { - if err := autotag.StudioScenes(studio, paths, r.Scene()); err != nil { + aliases, err := r.Studio().GetAliases(studio.ID) + if err != nil { return err } - if err := autotag.StudioImages(studio, paths, r.Image()); err != nil { + + if err := autotag.StudioScenes(studio, paths, aliases, r.Scene()); err != nil { return err } - if err := autotag.StudioGalleries(studio, paths, r.Gallery()); err != nil { + if err := autotag.StudioImages(studio, paths, aliases, r.Image()); err != nil { + return err + } + if err := autotag.StudioGalleries(studio, paths, aliases, r.Gallery()); err != nil { return err } @@ -380,9 +385,16 @@ func (t *autoTagFilesTask) makeImageFilter() *models.ImageFilterType { func (t *autoTagFilesTask) makeGalleryFilter() *models.GalleryFilterType { ret := &models.GalleryFilterType{} + or := ret sep := string(filepath.Separator) + if len(t.paths) == 0 { + ret.Path = &models.StringCriterionInput{ + Modifier: models.CriterionModifierNotNull, + } + } + for _, p := range t.paths { if !strings.HasSuffix(p, sep) { p = p + sep @@ -633,17 +645,17 @@ func (t *autoTagSceneTask) Start(wg *sync.WaitGroup) { if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if t.performers { if err := autotag.ScenePerformers(t.scene, r.Scene(), r.Performer()); err != nil { - return err + return fmt.Errorf("error tagging scene performers for %s: %v", t.scene.Path, err) } } if t.studios { if err := autotag.SceneStudios(t.scene, r.Scene(), r.Studio()); err != nil { - return err + return fmt.Errorf("error tagging scene studio for %s: %v", t.scene.Path, err) } } if t.tags { if err := autotag.SceneTags(t.scene, r.Scene(), r.Tag()); err != nil { - return err + return fmt.Errorf("error tagging scene tags for %s: %v", t.scene.Path, err) } } @@ -667,17 +679,17 @@ func (t *autoTagImageTask) Start(wg *sync.WaitGroup) { if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if t.performers { if err := autotag.ImagePerformers(t.image, r.Image(), r.Performer()); err != nil { - return err + return fmt.Errorf("error tagging image performers for %s: %v", t.image.Path, err) } } if t.studios { if err := autotag.ImageStudios(t.image, r.Image(), r.Studio()); err != nil { - return err + return fmt.Errorf("error tagging image studio for %s: %v", t.image.Path, err) } } if t.tags { if err := autotag.ImageTags(t.image, r.Image(), r.Tag()); err != nil { - return err + return fmt.Errorf("error tagging image tags for %s: %v", t.image.Path, err) } } @@ -701,17 +713,17 @@ func (t *autoTagGalleryTask) Start(wg *sync.WaitGroup) { if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if t.performers { if err := autotag.GalleryPerformers(t.gallery, r.Gallery(), r.Performer()); err != nil { - return err + return fmt.Errorf("error tagging gallery performers for %s: %v", t.gallery.Path.String, err) } } if t.studios { if err := autotag.GalleryStudios(t.gallery, r.Gallery(), r.Studio()); err != nil { - return err + return fmt.Errorf("error tagging gallery studio for %s: %v", t.gallery.Path.String, err) } } if t.tags { if err := autotag.GalleryTags(t.gallery, r.Gallery(), r.Tag()); err != nil { - return err + return fmt.Errorf("error tagging gallery tags for %s: %v", t.gallery.Path.String, err) } } diff --git a/pkg/manager/task_export.go b/pkg/manager/task_export.go index 65d925d1b..fbfda537c 100644 --- a/pkg/manager/task_export.go +++ b/pkg/manager/task_export.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "os" "path/filepath" "runtime" @@ -126,7 +125,7 @@ func (t *ExportTask) Start(wg *sync.WaitGroup) { paths.EnsureJSONDirs(t.baseDir) - t.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + txnErr := t.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { // include movie scenes and gallery images if !t.full { // only include movie scenes if includeDependencies is also set @@ -154,6 +153,9 @@ func (t *ExportTask) Start(wg *sync.WaitGroup) { return nil }) + if txnErr != nil { + logger.Warnf("error while running export transaction: %v", txnErr) + } if err := t.json.saveMappings(t.Mappings); err != nil { logger.Errorf("[mappings] failed to save json: %s", err.Error()) @@ -171,8 +173,10 @@ func (t *ExportTask) Start(wg *sync.WaitGroup) { func (t *ExportTask) generateDownload() error { // zip the files and register a download link - utils.EnsureDir(instance.Paths.Generated.Downloads) - z, err := ioutil.TempFile(instance.Paths.Generated.Downloads, "export*.zip") + if err := utils.EnsureDir(instance.Paths.Generated.Downloads); err != nil { + return err + } + z, err := os.CreateTemp(instance.Paths.Generated.Downloads, "export*.zip") if err != nil { return err } @@ -202,17 +206,24 @@ func (t *ExportTask) zipFiles(w io.Writer) error { return err } - filepath.Walk(t.json.json.Tags, t.zipWalkFunc(u.json.Tags, z)) - filepath.Walk(t.json.json.Galleries, t.zipWalkFunc(u.json.Galleries, z)) - filepath.Walk(t.json.json.Performers, t.zipWalkFunc(u.json.Performers, z)) - filepath.Walk(t.json.json.Studios, t.zipWalkFunc(u.json.Studios, z)) - filepath.Walk(t.json.json.Movies, t.zipWalkFunc(u.json.Movies, z)) - filepath.Walk(t.json.json.Scenes, t.zipWalkFunc(u.json.Scenes, z)) - filepath.Walk(t.json.json.Images, t.zipWalkFunc(u.json.Images, z)) + walkWarn(t.json.json.Tags, t.zipWalkFunc(u.json.Tags, z)) + walkWarn(t.json.json.Galleries, t.zipWalkFunc(u.json.Galleries, z)) + walkWarn(t.json.json.Performers, t.zipWalkFunc(u.json.Performers, z)) + walkWarn(t.json.json.Studios, t.zipWalkFunc(u.json.Studios, z)) + walkWarn(t.json.json.Movies, t.zipWalkFunc(u.json.Movies, z)) + walkWarn(t.json.json.Scenes, t.zipWalkFunc(u.json.Scenes, z)) + walkWarn(t.json.json.Images, t.zipWalkFunc(u.json.Images, z)) return nil } +// like filepath.Walk but issue a warning on error +func walkWarn(root string, fn filepath.WalkFunc) { + if err := filepath.Walk(root, fn); err != nil { + logger.Warnf("error walking structure %v: %v", root, err) + } +} + func (t *ExportTask) zipWalkFunc(outDir string, z *zip.Writer) filepath.WalkFunc { return func(path string, info os.FileInfo, err error) error { if err != nil { diff --git a/pkg/manager/task_generate_markers.go b/pkg/manager/task_generate_markers.go index 0e11fb853..a89a5e538 100644 --- a/pkg/manager/task_generate_markers.go +++ b/pkg/manager/task_generate_markers.go @@ -5,8 +5,6 @@ import ( "path/filepath" "strconv" - "github.com/remeh/sizedwaitgroup" - "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" @@ -19,11 +17,12 @@ type GenerateMarkersTask struct { Marker *models.SceneMarker Overwrite bool fileNamingAlgorithm models.HashAlgorithm + + ImagePreview bool + Screenshot bool } -func (t *GenerateMarkersTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { - defer wg.Done() - +func (t *GenerateMarkersTask) Start() { if t.Scene != nil { t.generateSceneMarkers() } @@ -79,7 +78,9 @@ func (t *GenerateMarkersTask) generateSceneMarkers() { // Make the folder for the scenes markers markersFolder := filepath.Join(instance.Paths.Generated.Markers, sceneHash) - utils.EnsureDir(markersFolder) + if err := utils.EnsureDir(markersFolder); err != nil { + logger.Warnf("could not create the markers folder (%v): %v", markersFolder, err) + } for i, sceneMarker := range sceneMarkers { index := i + 1 @@ -94,7 +95,8 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene seconds := int(sceneMarker.Seconds) videoExists := t.videoExists(sceneHash, seconds) - imageExists := t.imageExists(sceneHash, seconds) + imageExists := !t.ImagePreview || t.imageExists(sceneHash, seconds) + screenshotExists := !t.Screenshot || t.screenshotExists(sceneHash, seconds) baseFilename := strconv.Itoa(seconds) @@ -119,7 +121,7 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene } } - if t.Overwrite || !imageExists { + if t.ImagePreview && (t.Overwrite || !imageExists) { imageFilename := baseFilename + ".webp" imagePath := instance.Paths.SceneMarkers.GetStreamPreviewImagePath(sceneHash, seconds) @@ -131,6 +133,24 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene logger.Debug("created marker image: ", imagePath) } } + + if t.Screenshot && (t.Overwrite || !screenshotExists) { + screenshotFilename := baseFilename + ".jpg" + screenshotPath := instance.Paths.SceneMarkers.GetStreamScreenshotPath(sceneHash, seconds) + + screenshotOptions := ffmpeg.ScreenshotOptions{ + OutputPath: instance.Paths.Generated.GetTmpPath(screenshotFilename), // tmp output in case the process ends abruptly + Quality: 2, + Width: videoFile.Width, + Time: float64(seconds), + } + if err := encoder.Screenshot(*videoFile, screenshotOptions); err != nil { + logger.Errorf("[generator] failed to generate marker screenshot: %s", err) + } else { + _ = utils.SafeMove(screenshotOptions.OutputPath, screenshotPath) + logger.Debug("created marker screenshot: ", screenshotPath) + } + } } func (t *GenerateMarkersTask) isMarkerNeeded() int { @@ -166,12 +186,11 @@ func (t *GenerateMarkersTask) markerExists(sceneChecksum string, seconds int) bo return false } - videoPath := instance.Paths.SceneMarkers.GetStreamPath(sceneChecksum, seconds) - imagePath := instance.Paths.SceneMarkers.GetStreamPreviewImagePath(sceneChecksum, seconds) - videoExists, _ := utils.FileExists(videoPath) - imageExists, _ := utils.FileExists(imagePath) + videoExists := t.videoExists(sceneChecksum, seconds) + imageExists := !t.ImagePreview || t.imageExists(sceneChecksum, seconds) + screenshotExists := !t.Screenshot || t.screenshotExists(sceneChecksum, seconds) - return videoExists && imageExists + return videoExists && imageExists && screenshotExists } func (t *GenerateMarkersTask) videoExists(sceneChecksum string, seconds int) bool { @@ -195,3 +214,14 @@ func (t *GenerateMarkersTask) imageExists(sceneChecksum string, seconds int) boo return imageExists } + +func (t *GenerateMarkersTask) screenshotExists(sceneChecksum string, seconds int) bool { + if sceneChecksum == "" { + return false + } + + screenshotPath := instance.Paths.SceneMarkers.GetStreamScreenshotPath(sceneChecksum, seconds) + screenshotExists, _ := utils.FileExists(screenshotPath) + + return screenshotExists +} diff --git a/pkg/manager/task_generate_phash.go b/pkg/manager/task_generate_phash.go index 0c1578ee5..9d2fa172f 100644 --- a/pkg/manager/task_generate_phash.go +++ b/pkg/manager/task_generate_phash.go @@ -1,8 +1,6 @@ package manager import ( - "github.com/remeh/sizedwaitgroup" - "context" "database/sql" @@ -18,9 +16,7 @@ type GeneratePhashTask struct { txnManager models.TransactionManager } -func (t *GeneratePhashTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { - defer wg.Done() - +func (t *GeneratePhashTask) Start() { if !t.shouldGenerate() { return } diff --git a/pkg/manager/task_generate_preview.go b/pkg/manager/task_generate_preview.go index c5c666a6d..01a68f006 100644 --- a/pkg/manager/task_generate_preview.go +++ b/pkg/manager/task_generate_preview.go @@ -1,8 +1,6 @@ package manager import ( - "github.com/remeh/sizedwaitgroup" - "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/config" @@ -20,9 +18,7 @@ type GeneratePreviewTask struct { fileNamingAlgorithm models.HashAlgorithm } -func (t *GeneratePreviewTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { - defer wg.Done() - +func (t *GeneratePreviewTask) Start() { videoFilename := t.videoFilename() videoChecksum := t.Scene.GetHash(t.fileNamingAlgorithm) imageFilename := t.imageFilename() diff --git a/pkg/manager/task_generate_screenshot.go b/pkg/manager/task_generate_screenshot.go index 21a340a92..8694c2357 100644 --- a/pkg/manager/task_generate_screenshot.go +++ b/pkg/manager/task_generate_screenshot.go @@ -3,9 +3,8 @@ package manager import ( "context" "fmt" - "io/ioutil" + "io" "os" - "sync" "time" "github.com/stashapp/stash/pkg/ffmpeg" @@ -20,9 +19,7 @@ type GenerateScreenshotTask struct { txnManager models.TransactionManager } -func (t *GenerateScreenshotTask) Start(wg *sync.WaitGroup) { - defer wg.Done() - +func (t *GenerateScreenshotTask) Start() { scenePath := t.Scene.Path probeResult, err := ffmpeg.NewVideoFile(instance.FFProbePath, scenePath, false) @@ -55,7 +52,7 @@ func (t *GenerateScreenshotTask) Start(wg *sync.WaitGroup) { } defer f.Close() - coverImageData, err := ioutil.ReadAll(f) + coverImageData, err := io.ReadAll(f) if err != nil { logger.Errorf("Error reading screenshot: %s", err.Error()) return @@ -70,18 +67,18 @@ func (t *GenerateScreenshotTask) Start(wg *sync.WaitGroup) { } if err := SetSceneScreenshot(checksum, coverImageData); err != nil { - return fmt.Errorf("Error writing screenshot: %s", err.Error()) + return fmt.Errorf("error writing screenshot: %v", err) } // update the scene cover table if err := qb.UpdateCover(t.Scene.ID, coverImageData); err != nil { - return fmt.Errorf("Error setting screenshot: %s", err.Error()) + return fmt.Errorf("error setting screenshot: %v", err) } // update the scene with the update date _, err = qb.Update(updatedScene) if err != nil { - return fmt.Errorf("Error updating scene: %s", err.Error()) + return fmt.Errorf("error updating scene: %v", err) } return nil diff --git a/pkg/manager/task_generate_sprite.go b/pkg/manager/task_generate_sprite.go index 20caeb78c..0a21b6011 100644 --- a/pkg/manager/task_generate_sprite.go +++ b/pkg/manager/task_generate_sprite.go @@ -1,8 +1,6 @@ package manager import ( - "github.com/remeh/sizedwaitgroup" - "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" @@ -15,9 +13,7 @@ type GenerateSpriteTask struct { fileNamingAlgorithm models.HashAlgorithm } -func (t *GenerateSpriteTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { - defer wg.Done() - +func (t *GenerateSpriteTask) Start() { if !t.Overwrite && !t.required() { return } diff --git a/pkg/manager/task_import.go b/pkg/manager/task_import.go index 84acaaeef..4d9b535ae 100644 --- a/pkg/manager/task_import.go +++ b/pkg/manager/task_import.go @@ -8,7 +8,6 @@ import ( "io" "os" "path/filepath" - "sync" "time" "github.com/stashapp/stash/pkg/database" @@ -79,9 +78,7 @@ func (t *ImportTask) GetDescription() string { return "Importing..." } -func (t *ImportTask) Start(wg *sync.WaitGroup) { - defer wg.Done() - +func (t *ImportTask) Start() { if t.TmpZip != "" { defer func() { err := utils.RemoveDir(t.BaseDir) @@ -160,7 +157,9 @@ func (t *ImportTask) unzipFile() error { fn := filepath.Join(t.BaseDir, f.Name) if f.FileInfo().IsDir() { - os.MkdirAll(fn, os.ModePerm) + if err := os.MkdirAll(fn, os.ModePerm); err != nil { + logger.Warnf("couldn't create directory %v while unzipping import file: %v", fn, err) + } continue } @@ -376,6 +375,7 @@ func (t *ImportTask) ImportGalleries(ctx context.Context) { } func (t *ImportTask) ImportTags(ctx context.Context) { + pendingParent := make(map[string][]*jsonschema.Tag) logger.Info("[tags] importing") for i, mappingJSON := range t.mappings.Tags { @@ -389,23 +389,64 @@ func (t *ImportTask) ImportTags(ctx context.Context) { logger.Progressf("[tags] %d of %d", index, len(t.mappings.Tags)) if err := t.txnManager.WithTxn(ctx, func(r models.Repository) error { - readerWriter := r.Tag() - - tagImporter := &tag.Importer{ - ReaderWriter: readerWriter, - Input: *tagJSON, + return t.ImportTag(tagJSON, pendingParent, false, r.Tag()) + }); err != nil { + if parentError, ok := err.(tag.ParentTagNotExistError); ok { + pendingParent[parentError.MissingParent()] = append(pendingParent[parentError.MissingParent()], tagJSON) + continue } - return performImport(tagImporter, t.DuplicateBehaviour) - }); err != nil { logger.Errorf("[tags] <%s> failed to import: %s", mappingJSON.Checksum, err.Error()) continue } } + for _, s := range pendingParent { + for _, orphanTagJSON := range s { + if err := t.txnManager.WithTxn(ctx, func(r models.Repository) error { + return t.ImportTag(orphanTagJSON, nil, true, r.Tag()) + }); err != nil { + logger.Errorf("[tags] <%s> failed to create: %s", orphanTagJSON.Name, err.Error()) + continue + } + } + } + logger.Info("[tags] import complete") } +func (t *ImportTask) ImportTag(tagJSON *jsonschema.Tag, pendingParent map[string][]*jsonschema.Tag, fail bool, readerWriter models.TagReaderWriter) error { + importer := &tag.Importer{ + ReaderWriter: readerWriter, + Input: *tagJSON, + MissingRefBehaviour: t.MissingRefBehaviour, + } + + // first phase: return error if parent does not exist + if !fail { + importer.MissingRefBehaviour = models.ImportMissingRefEnumFail + } + + if err := performImport(importer, t.DuplicateBehaviour); err != nil { + return err + } + + for _, childTagJSON := range pendingParent[tagJSON.Name] { + if err := t.ImportTag(childTagJSON, pendingParent, fail, readerWriter); err != nil { + if parentError, ok := err.(tag.ParentTagNotExistError); ok { + pendingParent[parentError.MissingParent()] = append(pendingParent[parentError.MissingParent()], tagJSON) + continue + } + + return fmt.Errorf("failed to create child tag <%s>: %s", childTagJSON.Name, err.Error()) + } + } + + delete(pendingParent, tagJSON.Name) + + return nil +} + func (t *ImportTask) ImportScrapedItems(ctx context.Context) { if err := t.txnManager.WithTxn(ctx, func(r models.Repository) error { logger.Info("[scraped sites] importing") @@ -569,101 +610,6 @@ func (t *ImportTask) ImportImages(ctx context.Context) { logger.Info("[images] import complete") } -func (t *ImportTask) getPerformers(names []string, qb models.PerformerReader) ([]*models.Performer, error) { - performers, err := qb.FindByNames(names, false) - if err != nil { - return nil, err - } - - var pluckedNames []string - for _, performer := range performers { - if !performer.Name.Valid { - continue - } - pluckedNames = append(pluckedNames, performer.Name.String) - } - - missingPerformers := utils.StrFilter(names, func(name string) bool { - return !utils.StrInclude(pluckedNames, name) - }) - - for _, missingPerformer := range missingPerformers { - logger.Warnf("[scenes] performer %s does not exist", missingPerformer) - } - - return performers, nil -} - -func (t *ImportTask) getMoviesScenes(input []jsonschema.SceneMovie, sceneID int, mqb models.MovieReader) ([]models.MoviesScenes, error) { - var movies []models.MoviesScenes - for _, inputMovie := range input { - movie, err := mqb.FindByName(inputMovie.MovieName, false) - if err != nil { - return nil, err - } - - if movie == nil { - logger.Warnf("[scenes] movie %s does not exist", inputMovie.MovieName) - } else { - toAdd := models.MoviesScenes{ - MovieID: movie.ID, - SceneID: sceneID, - } - - if inputMovie.SceneIndex != 0 { - toAdd.SceneIndex = sql.NullInt64{ - Int64: int64(inputMovie.SceneIndex), - Valid: true, - } - } - - movies = append(movies, toAdd) - } - } - - return movies, nil -} - -func (t *ImportTask) getTags(sceneChecksum string, names []string, tqb models.TagReader) ([]*models.Tag, error) { - tags, err := tqb.FindByNames(names, false) - if err != nil { - return nil, err - } - - var pluckedNames []string - for _, tag := range tags { - if tag.Name == "" { - continue - } - pluckedNames = append(pluckedNames, tag.Name) - } - - missingTags := utils.StrFilter(names, func(name string) bool { - return !utils.StrInclude(pluckedNames, name) - }) - - for _, missingTag := range missingTags { - logger.Warnf("[scenes] <%s> tag %s does not exist", sceneChecksum, missingTag) - } - - return tags, nil -} - -// https://www.reddit.com/r/golang/comments/5ia523/idiomatic_way_to_remove_duplicates_in_a_slice/db6qa2e -func (t *ImportTask) getUnique(s []string) []string { - seen := make(map[string]struct{}, len(s)) - j := 0 - for _, v := range s { - if _, ok := seen[v]; ok { - continue - } - seen[v] = struct{}{} - s[j] = v - j++ - } - return s[:j] -} - var currentLocation = time.Now().Location() func (t *ImportTask) getTimeFromJSONTime(jsonTime models.JSONTime) time.Time { diff --git a/pkg/manager/task_migrate_hash.go b/pkg/manager/task_migrate_hash.go index 3305ad9f6..3ecdb54d4 100644 --- a/pkg/manager/task_migrate_hash.go +++ b/pkg/manager/task_migrate_hash.go @@ -1,8 +1,6 @@ package manager import ( - "sync" - "github.com/stashapp/stash/pkg/models" ) @@ -14,9 +12,7 @@ type MigrateHashTask struct { } // Start starts the task. -func (t *MigrateHashTask) Start(wg *sync.WaitGroup) { - defer wg.Done() - +func (t *MigrateHashTask) Start() { if !t.Scene.OSHash.Valid || !t.Scene.Checksum.Valid { // nothing to do return diff --git a/pkg/manager/task_scan.go b/pkg/manager/task_scan.go index c90157611..bec812a7b 100644 --- a/pkg/manager/task_scan.go +++ b/pkg/manager/task_scan.go @@ -83,11 +83,23 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) { return stoppingErr } + // #1756 - skip zero length files and directories + if info.IsDir() { + return nil + } + + if info.Size() == 0 { + logger.Infof("Skipping zero-length file: %s", path) + return nil + } + if isGallery(path) { galleries = append(galleries, path) } - instance.Paths.Generated.EnsureTmpDir() + if err := instance.Paths.Generated.EnsureTmpDir(); err != nil { + logger.Warnf("couldn't create temporary directory: %v", err) + } wg.Add() task := ScanTask{ @@ -101,13 +113,15 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) { GenerateImagePreview: utils.IsTrue(input.ScanGenerateImagePreviews), GenerateSprite: utils.IsTrue(input.ScanGenerateSprites), GeneratePhash: utils.IsTrue(input.ScanGeneratePhashes), + GenerateThumbnails: utils.IsTrue(input.ScanGenerateThumbnails), progress: progress, CaseSensitiveFs: csFs, ctx: ctx, } go func() { - task.Start(&wg) + task.Start() + wg.Done() progress.Increment() }() @@ -126,7 +140,11 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) { } wg.Wait() - instance.Paths.Generated.EmptyTmpDir() + + if err := instance.Paths.Generated.EmptyTmpDir(); err != nil { + logger.Warnf("couldn't empty temporary directory: %v", err) + } + elapsed := time.Since(start) logger.Info(fmt.Sprintf("Scan finished (%s)", elapsed)) @@ -214,14 +232,13 @@ type ScanTask struct { GeneratePhash bool GeneratePreview bool GenerateImagePreview bool + GenerateThumbnails bool zipGallery *models.Gallery progress *job.Progress CaseSensitiveFs bool } -func (t *ScanTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { - defer wg.Done() - +func (t *ScanTask) Start() { var s *models.Scene t.progress.ExecuteTask("Scanning "+t.FilePath, func() { @@ -246,7 +263,8 @@ func (t *ScanTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { Overwrite: false, fileNamingAlgorithm: t.fileNamingAlgorithm, } - taskSprite.Start(&iwg) + taskSprite.Start() + iwg.Done() }) } @@ -259,7 +277,8 @@ func (t *ScanTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { fileNamingAlgorithm: t.fileNamingAlgorithm, txnManager: t.TxnManager, } - taskPhash.Start(&iwg) + taskPhash.Start() + iwg.Done() }) } @@ -290,7 +309,8 @@ func (t *ScanTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { Overwrite: false, fileNamingAlgorithm: t.fileNamingAlgorithm, } - taskPreview.Start(wg) + taskPreview.Start() + iwg.Done() }) } @@ -391,17 +411,13 @@ func (t *ScanTask) scanGallery() { // scan the zip files if the gallery has no images scanImages = scanImages || images == 0 } else { - // Ignore directories. - if isDir, _ := utils.DirExists(t.FilePath); isDir { - return - } - checksum, err := t.calculateChecksum() if err != nil { logger.Error(err.Error()) return } + isNewGallery := false if err := t.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { qb := r.Gallery() g, _ = qb.FindByChecksum(checksum) @@ -467,7 +483,7 @@ func (t *ScanTask) scanGallery() { } scanImages = true - GetInstance().PluginCache.ExecutePostHooks(t.ctx, g.ID, plugin.GalleryCreatePost, nil, nil) + isNewGallery = true } } @@ -476,6 +492,10 @@ func (t *ScanTask) scanGallery() { logger.Error(err.Error()) return } + + if isNewGallery { + GetInstance().PluginCache.ExecutePostHooks(t.ctx, g.ID, plugin.GalleryCreatePost, nil, nil) + } } if g != nil { @@ -714,11 +734,6 @@ func (t *ScanTask) scanScene() *models.Scene { return nil } - // Ignore directories. - if isDir, _ := utils.DirExists(t.FilePath); isDir { - return nil - } - videoFile, err := ffmpeg.NewVideoFile(instance.FFProbePath, t.FilePath, t.StripFileExtension) if err != nil { logger.Error(err.Error()) @@ -748,7 +763,7 @@ func (t *ScanTask) scanScene() *models.Scene { // check for scene by checksum and oshash - MD5 should be // redundant, but check both - t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + txnErr := t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { qb := r.Scene() if checksum != "" { s, _ = qb.FindByChecksum(checksum) @@ -760,6 +775,9 @@ func (t *ScanTask) scanScene() *models.Scene { return nil }) + if txnErr != nil { + logger.Warnf("error in read transaction: %v", txnErr) + } sceneHash := oshash @@ -958,9 +976,7 @@ func (t *ScanTask) scanZipImages(zipGallery *models.Gallery) { subTask.zipGallery = zipGallery // run the subtask and wait for it to complete - iwg := sizedwaitgroup.New(1) - iwg.Add() - subTask.Start(&iwg) + subTask.Start() return nil }) if err != nil { @@ -1043,11 +1059,6 @@ func (t *ScanTask) scanImage() { // check for thumbnails t.generateThumbnail(i) } else { - // Ignore directories. - if isDir, _ := utils.DirExists(t.FilePath); isDir { - return - } - var checksum string logger.Infof("%s not found. Calculating checksum...", t.FilePath) @@ -1141,12 +1152,20 @@ func (t *ScanTask) scanImage() { } else if config.GetInstance().GetCreateGalleriesFromFolders() { // create gallery from folder or associate with existing gallery logger.Infof("Associating image %s with folder gallery", i.Path) + var galleryID int + var isNewGallery bool if err := t.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { - return t.associateImageWithFolderGallery(i.ID, r.Gallery()) + var err error + galleryID, isNewGallery, err = t.associateImageWithFolderGallery(i.ID, r.Gallery()) + return err }); err != nil { logger.Error(err.Error()) return } + + if isNewGallery { + GetInstance().PluginCache.ExecutePostHooks(t.ctx, galleryID, plugin.GalleryCreatePost, nil, nil) + } } } @@ -1208,12 +1227,13 @@ func (t *ScanTask) rescanImage(i *models.Image, fileModTime time.Time) (*models. return ret, nil } -func (t *ScanTask) associateImageWithFolderGallery(imageID int, qb models.GalleryReaderWriter) error { +func (t *ScanTask) associateImageWithFolderGallery(imageID int, qb models.GalleryReaderWriter) (galleryID int, isNew bool, err error) { // find a gallery with the path specified path := filepath.Dir(t.FilePath) - g, err := qb.FindByPath(path) + var g *models.Gallery + g, err = qb.FindByPath(path) if err != nil { - return err + return } if g == nil { @@ -1239,30 +1259,39 @@ func (t *ScanTask) associateImageWithFolderGallery(imageID int, qb models.Galler logger.Infof("Creating gallery for folder %s", path) g, err = qb.Create(newGallery) if err != nil { - return err + return 0, false, err } + + isNew = true } // associate image with gallery err = gallery.AddImage(qb, g.ID, imageID) - return err + galleryID = g.ID + return } func (t *ScanTask) generateThumbnail(i *models.Image) { + if !t.GenerateThumbnails { + return + } + thumbPath := GetInstance().Paths.Generated.GetThumbnailPath(i.Checksum, models.DefaultGthumbWidth) exists, _ := utils.FileExists(thumbPath) if exists { return } - srcImage, err := image.GetSourceImage(i) + config, _, err := image.DecodeSourceImage(i) if err != nil { logger.Errorf("error reading image %s: %s", i.Path, err.Error()) return } - if image.ThumbnailNeeded(srcImage, models.DefaultGthumbWidth) { - data, err := image.GetThumbnail(srcImage, models.DefaultGthumbWidth) + if config.Height > models.DefaultGthumbWidth || config.Width > models.DefaultGthumbWidth { + encoder := image.NewThumbnailEncoder(instance.FFMPEGPath) + data, err := encoder.GetThumbnail(i, models.DefaultGthumbWidth) + if err != nil { logger.Errorf("error getting thumbnail for image %s: %s", i.Path, err.Error()) return @@ -1303,7 +1332,7 @@ func (t *ScanTask) doesPathExist() bool { gExt := config.GetGalleryExtensions() ret := false - t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + txnErr := t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if matchExtension(t.FilePath, gExt) { gallery, _ := r.Gallery().FindByPath(t.FilePath) if gallery != nil { @@ -1323,6 +1352,9 @@ func (t *ScanTask) doesPathExist() bool { return nil }) + if txnErr != nil { + logger.Warnf("error while executing read transaction: %v", txnErr) + } return ret } diff --git a/pkg/manager/task_stash_box_tag.go b/pkg/manager/task_stash_box_tag.go index 597655e9f..7a7e7b8f7 100644 --- a/pkg/manager/task_stash_box_tag.go +++ b/pkg/manager/task_stash_box_tag.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "sync" "time" "github.com/stashapp/stash/pkg/logger" @@ -22,9 +21,7 @@ type StashBoxPerformerTagTask struct { excluded_fields []string } -func (t *StashBoxPerformerTagTask) Start(wg *sync.WaitGroup) { - defer wg.Done() - +func (t *StashBoxPerformerTagTask) Start() { t.stashBoxPerformerTag() } @@ -40,14 +37,14 @@ func (t *StashBoxPerformerTagTask) Description() string { } func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { - var performer *models.ScrapedScenePerformer + var performer *models.ScrapedPerformer var err error client := stashbox.NewClient(*t.box, t.txnManager) if t.refresh { var performerID string - t.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + txnErr := t.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { stashids, _ := r.Performer().GetStashIDs(t.performer.ID) for _, id := range stashids { if id.Endpoint == t.box.Endpoint { @@ -56,6 +53,9 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { } return nil }) + if txnErr != nil { + logger.Warnf("error while executing read transaction: %v", err) + } if performerID != "" { performer, err = client.FindStashBoxPerformerByID(performerID) } @@ -132,8 +132,8 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { value := getNullString(performer.Measurements) partial.Measurements = &value } - if excluded["name"] { - value := sql.NullString{String: performer.Name, Valid: true} + if excluded["name"] && performer.Name != nil { + value := sql.NullString{String: *performer.Name, Valid: true} partial.Name = &value } if performer.Piercings != nil && !excluded["piercings"] { @@ -153,7 +153,7 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { partial.URL = &value } - t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { + txnErr := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { _, err := r.Performer().Update(partial) if !t.refresh { @@ -180,17 +180,24 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { } if err == nil { - logger.Infof("Updated performer %s", performer.Name) + var name string + if performer.Name != nil { + name = *performer.Name + } + logger.Infof("Updated performer %s", name) } return err }) - } else if t.name != nil { + if txnErr != nil { + logger.Warnf("failure to execute partial update of performer: %v", err) + } + } else if t.name != nil && performer.Name != nil { currentTime := time.Now() newPerformer := models.Performer{ Aliases: getNullString(performer.Aliases), Birthdate: getDate(performer.Birthdate), CareerLength: getNullString(performer.CareerLength), - Checksum: utils.MD5FromString(performer.Name), + Checksum: utils.MD5FromString(*performer.Name), Country: getNullString(performer.Country), CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, Ethnicity: getNullString(performer.Ethnicity), @@ -201,7 +208,7 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { Height: getNullString(performer.Height), Instagram: getNullString(performer.Instagram), Measurements: getNullString(performer.Measurements), - Name: sql.NullString{String: performer.Name, Valid: true}, + Name: sql.NullString{String: *performer.Name, Valid: true}, Piercings: getNullString(performer.Piercings), Tattoos: getNullString(performer.Tattoos), Twitter: getNullString(performer.Twitter), @@ -225,9 +232,9 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { } if len(performer.Images) > 0 { - image, err := utils.ReadImageFromURL(performer.Images[0]) - if err != nil { - return err + image, imageErr := utils.ReadImageFromURL(performer.Images[0]) + if imageErr != nil { + return imageErr } err = r.Performer().UpdateImage(createdPerformer.ID, image) } diff --git a/pkg/manager/task_transcode.go b/pkg/manager/task_transcode.go index 5a9161966..7c55eaba5 100644 --- a/pkg/manager/task_transcode.go +++ b/pkg/manager/task_transcode.go @@ -1,8 +1,6 @@ package manager import ( - "github.com/remeh/sizedwaitgroup" - "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/config" @@ -16,9 +14,7 @@ type GenerateTranscodeTask struct { fileNamingAlgorithm models.HashAlgorithm } -func (t *GenerateTranscodeTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { - defer wg.Done() - +func (t *GenerateTranscodeTask) Start() { hasTranscode := HasTranscode(&t.Scene, t.fileNamingAlgorithm) if !t.Overwrite && hasTranscode { return diff --git a/pkg/models/mocks/MovieReaderWriter.go b/pkg/models/mocks/MovieReaderWriter.go index 8cf71e4a5..3f80f12a3 100644 --- a/pkg/models/mocks/MovieReaderWriter.go +++ b/pkg/models/mocks/MovieReaderWriter.go @@ -56,6 +56,48 @@ func (_m *MovieReaderWriter) Count() (int, error) { return r0, r1 } +// CountByPerformerID provides a mock function with given fields: performerID +func (_m *MovieReaderWriter) CountByPerformerID(performerID int) (int, error) { + ret := _m.Called(performerID) + + var r0 int + if rf, ok := ret.Get(0).(func(int) int); ok { + r0 = rf(performerID) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(performerID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CountByStudioID provides a mock function with given fields: studioID +func (_m *MovieReaderWriter) CountByStudioID(studioID int) (int, error) { + ret := _m.Called(studioID) + + var r0 int + if rf, ok := ret.Get(0).(func(int) int); ok { + r0 = rf(studioID) + } else { + r0 = ret.Get(0).(int) + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(studioID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Create provides a mock function with given fields: newMovie func (_m *MovieReaderWriter) Create(newMovie models.Movie) (*models.Movie, error) { ret := _m.Called(newMovie) @@ -176,6 +218,52 @@ func (_m *MovieReaderWriter) FindByNames(names []string, nocase bool) ([]*models return r0, r1 } +// FindByPerformerID provides a mock function with given fields: performerID +func (_m *MovieReaderWriter) FindByPerformerID(performerID int) ([]*models.Movie, error) { + ret := _m.Called(performerID) + + var r0 []*models.Movie + if rf, ok := ret.Get(0).(func(int) []*models.Movie); ok { + r0 = rf(performerID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Movie) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(performerID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindByStudioID provides a mock function with given fields: studioID +func (_m *MovieReaderWriter) FindByStudioID(studioID int) ([]*models.Movie, error) { + ret := _m.Called(studioID) + + var r0 []*models.Movie + if rf, ok := ret.Get(0).(func(int) []*models.Movie); ok { + r0 = rf(studioID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Movie) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(studioID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // FindMany provides a mock function with given fields: ids func (_m *MovieReaderWriter) FindMany(ids []int) ([]*models.Movie, error) { ret := _m.Called(ids) diff --git a/pkg/models/mocks/StudioReaderWriter.go b/pkg/models/mocks/StudioReaderWriter.go index fbd8a1936..3c7b61ab0 100644 --- a/pkg/models/mocks/StudioReaderWriter.go +++ b/pkg/models/mocks/StudioReaderWriter.go @@ -199,6 +199,29 @@ func (_m *StudioReaderWriter) FindMany(ids []int) ([]*models.Studio, error) { return r0, r1 } +// GetAliases provides a mock function with given fields: studioID +func (_m *StudioReaderWriter) GetAliases(studioID int) ([]string, error) { + ret := _m.Called(studioID) + + var r0 []string + if rf, ok := ret.Get(0).(func(int) []string); ok { + r0 = rf(studioID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(studioID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetImage provides a mock function with given fields: studioID func (_m *StudioReaderWriter) GetImage(studioID int) ([]byte, error) { ret := _m.Called(studioID) @@ -342,6 +365,20 @@ func (_m *StudioReaderWriter) Update(updatedStudio models.StudioPartial) (*model return r0, r1 } +// UpdateAliases provides a mock function with given fields: studioID, aliases +func (_m *StudioReaderWriter) UpdateAliases(studioID int, aliases []string) error { + ret := _m.Called(studioID, aliases) + + var r0 error + if rf, ok := ret.Get(0).(func(int, []string) error); ok { + r0 = rf(studioID, aliases) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // UpdateFull provides a mock function with given fields: updatedStudio func (_m *StudioReaderWriter) UpdateFull(updatedStudio models.Studio) (*models.Studio, error) { ret := _m.Called(updatedStudio) diff --git a/pkg/models/mocks/TagReaderWriter.go b/pkg/models/mocks/TagReaderWriter.go index f7cabb8ce..f6b257ed2 100644 --- a/pkg/models/mocks/TagReaderWriter.go +++ b/pkg/models/mocks/TagReaderWriter.go @@ -130,6 +130,75 @@ func (_m *TagReaderWriter) Find(id int) (*models.Tag, error) { return r0, r1 } +// FindAllAncestors provides a mock function with given fields: tagID, excludeIDs +func (_m *TagReaderWriter) FindAllAncestors(tagID int, excludeIDs []int) ([]*models.Tag, error) { + ret := _m.Called(tagID, excludeIDs) + + var r0 []*models.Tag + if rf, ok := ret.Get(0).(func(int, []int) []*models.Tag); ok { + r0 = rf(tagID, excludeIDs) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Tag) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int, []int) error); ok { + r1 = rf(tagID, excludeIDs) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindAllDescendants provides a mock function with given fields: tagID, excludeIDs +func (_m *TagReaderWriter) FindAllDescendants(tagID int, excludeIDs []int) ([]*models.Tag, error) { + ret := _m.Called(tagID, excludeIDs) + + var r0 []*models.Tag + if rf, ok := ret.Get(0).(func(int, []int) []*models.Tag); ok { + r0 = rf(tagID, excludeIDs) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Tag) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int, []int) error); ok { + r1 = rf(tagID, excludeIDs) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FindByChildTagID provides a mock function with given fields: childID +func (_m *TagReaderWriter) FindByChildTagID(childID int) ([]*models.Tag, error) { + ret := _m.Called(childID) + + var r0 []*models.Tag + if rf, ok := ret.Get(0).(func(int) []*models.Tag); ok { + r0 = rf(childID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Tag) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(childID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // FindByGalleryID provides a mock function with given fields: galleryID func (_m *TagReaderWriter) FindByGalleryID(galleryID int) ([]*models.Tag, error) { ret := _m.Called(galleryID) @@ -222,6 +291,29 @@ func (_m *TagReaderWriter) FindByNames(names []string, nocase bool) ([]*models.T return r0, r1 } +// FindByParentTagID provides a mock function with given fields: parentID +func (_m *TagReaderWriter) FindByParentTagID(parentID int) ([]*models.Tag, error) { + ret := _m.Called(parentID) + + var r0 []*models.Tag + if rf, ok := ret.Get(0).(func(int) []*models.Tag); ok { + r0 = rf(parentID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Tag) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int) error); ok { + r1 = rf(parentID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // FindByPerformerID provides a mock function with given fields: performerID func (_m *TagReaderWriter) FindByPerformerID(performerID int) ([]*models.Tag, error) { ret := _m.Called(performerID) @@ -464,6 +556,20 @@ func (_m *TagReaderWriter) UpdateAliases(tagID int, aliases []string) error { return r0 } +// UpdateChildTags provides a mock function with given fields: tagID, parentIDs +func (_m *TagReaderWriter) UpdateChildTags(tagID int, parentIDs []int) error { + ret := _m.Called(tagID, parentIDs) + + var r0 error + if rf, ok := ret.Get(0).(func(int, []int) error); ok { + r0 = rf(tagID, parentIDs) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // UpdateFull provides a mock function with given fields: updatedTag func (_m *TagReaderWriter) UpdateFull(updatedTag models.Tag) (*models.Tag, error) { ret := _m.Called(updatedTag) @@ -500,3 +606,17 @@ func (_m *TagReaderWriter) UpdateImage(tagID int, image []byte) error { return r0 } + +// UpdateParentTags provides a mock function with given fields: tagID, parentIDs +func (_m *TagReaderWriter) UpdateParentTags(tagID int, parentIDs []int) error { + ret := _m.Called(tagID, parentIDs) + + var r0 error + if rf, ok := ret.Get(0).(func(int, []int) error); ok { + r0 = rf(tagID, parentIDs) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/pkg/models/model_scraped_item.go b/pkg/models/model_scraped_item.go index 230fd0ba0..4035163b7 100644 --- a/pkg/models/model_scraped_item.go +++ b/pkg/models/model_scraped_item.go @@ -23,174 +23,6 @@ type ScrapedItem struct { UpdatedAt SQLiteTimestamp `db:"updated_at" json:"updated_at"` } -type ScrapedPerformer struct { - Name *string `graphql:"name" json:"name"` - Gender *string `graphql:"gender" json:"gender"` - URL *string `graphql:"url" json:"url"` - Twitter *string `graphql:"twitter" json:"twitter"` - Instagram *string `graphql:"instagram" json:"instagram"` - Birthdate *string `graphql:"birthdate" json:"birthdate"` - Ethnicity *string `graphql:"ethnicity" json:"ethnicity"` - Country *string `graphql:"country" json:"country"` - EyeColor *string `graphql:"eye_color" json:"eye_color"` - Height *string `graphql:"height" json:"height"` - Measurements *string `graphql:"measurements" json:"measurements"` - FakeTits *string `graphql:"fake_tits" json:"fake_tits"` - CareerLength *string `graphql:"career_length" json:"career_length"` - Tattoos *string `graphql:"tattoos" json:"tattoos"` - Piercings *string `graphql:"piercings" json:"piercings"` - Aliases *string `graphql:"aliases" json:"aliases"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - Image *string `graphql:"image" json:"image"` - Details *string `graphql:"details" json:"details"` - DeathDate *string `graphql:"death_date" json:"death_date"` - HairColor *string `graphql:"hair_color" json:"hair_color"` - Weight *string `graphql:"weight" json:"weight"` - RemoteSiteID *string `graphql:"remote_site_id" json:"remote_site_id"` -} - -// this type has no Image field -type ScrapedPerformerStash struct { - Name *string `graphql:"name" json:"name"` - Gender *string `graphql:"gender" json:"gender"` - URL *string `graphql:"url" json:"url"` - Twitter *string `graphql:"twitter" json:"twitter"` - Instagram *string `graphql:"instagram" json:"instagram"` - Birthdate *string `graphql:"birthdate" json:"birthdate"` - Ethnicity *string `graphql:"ethnicity" json:"ethnicity"` - Country *string `graphql:"country" json:"country"` - EyeColor *string `graphql:"eye_color" json:"eye_color"` - Height *string `graphql:"height" json:"height"` - Measurements *string `graphql:"measurements" json:"measurements"` - FakeTits *string `graphql:"fake_tits" json:"fake_tits"` - CareerLength *string `graphql:"career_length" json:"career_length"` - Tattoos *string `graphql:"tattoos" json:"tattoos"` - Piercings *string `graphql:"piercings" json:"piercings"` - Aliases *string `graphql:"aliases" json:"aliases"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - Details *string `graphql:"details" json:"details"` - DeathDate *string `graphql:"death_date" json:"death_date"` - HairColor *string `graphql:"hair_color" json:"hair_color"` - Weight *string `graphql:"weight" json:"weight"` -} - -type ScrapedScene struct { - Title *string `graphql:"title" json:"title"` - Details *string `graphql:"details" json:"details"` - URL *string `graphql:"url" json:"url"` - Date *string `graphql:"date" json:"date"` - Image *string `graphql:"image" json:"image"` - RemoteSiteID *string `graphql:"remote_site_id" json:"remote_site_id"` - Duration *int `graphql:"duration" json:"duration"` - File *SceneFileType `graphql:"file" json:"file"` - Fingerprints []*StashBoxFingerprint `graphql:"fingerprints" json:"fingerprints"` - Studio *ScrapedSceneStudio `graphql:"studio" json:"studio"` - Movies []*ScrapedSceneMovie `graphql:"movies" json:"movies"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - Performers []*ScrapedScenePerformer `graphql:"performers" json:"performers"` -} - -// stash doesn't return image, and we need id -type ScrapedSceneStash struct { - ID string `graphql:"id" json:"id"` - Title *string `graphql:"title" json:"title"` - Details *string `graphql:"details" json:"details"` - URL *string `graphql:"url" json:"url"` - Date *string `graphql:"date" json:"date"` - File *SceneFileType `graphql:"file" json:"file"` - Studio *ScrapedSceneStudio `graphql:"studio" json:"studio"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - Performers []*ScrapedScenePerformer `graphql:"performers" json:"performers"` -} - -type ScrapedGalleryStash struct { - ID string `graphql:"id" json:"id"` - Title *string `graphql:"title" json:"title"` - Details *string `graphql:"details" json:"details"` - URL *string `graphql:"url" json:"url"` - Date *string `graphql:"date" json:"date"` - File *SceneFileType `graphql:"file" json:"file"` - Studio *ScrapedSceneStudio `graphql:"studio" json:"studio"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - Performers []*ScrapedScenePerformer `graphql:"performers" json:"performers"` -} - -type ScrapedScenePerformer struct { - // Set if performer matched - ID *string `graphql:"id" json:"id"` - Name string `graphql:"name" json:"name"` - Gender *string `graphql:"gender" json:"gender"` - URL *string `graphql:"url" json:"url"` - Twitter *string `graphql:"twitter" json:"twitter"` - Instagram *string `graphql:"instagram" json:"instagram"` - Birthdate *string `graphql:"birthdate" json:"birthdate"` - Ethnicity *string `graphql:"ethnicity" json:"ethnicity"` - Country *string `graphql:"country" json:"country"` - EyeColor *string `graphql:"eye_color" json:"eye_color"` - Height *string `graphql:"height" json:"height"` - Measurements *string `graphql:"measurements" json:"measurements"` - FakeTits *string `graphql:"fake_tits" json:"fake_tits"` - CareerLength *string `graphql:"career_length" json:"career_length"` - Tattoos *string `graphql:"tattoos" json:"tattoos"` - Piercings *string `graphql:"piercings" json:"piercings"` - Aliases *string `graphql:"aliases" json:"aliases"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - RemoteSiteID *string `graphql:"remote_site_id" json:"remote_site_id"` - Images []string `graphql:"images" json:"images"` - Details *string `graphql:"details" json:"details"` - DeathDate *string `graphql:"death_date" json:"death_date"` - HairColor *string `graphql:"hair_color" json:"hair_color"` - Weight *string `graphql:"weight" json:"weight"` -} - -type ScrapedSceneStudio struct { - // Set if studio matched - ID *string `graphql:"id" json:"id"` - Name string `graphql:"name" json:"name"` - URL *string `graphql:"url" json:"url"` - RemoteSiteID *string `graphql:"remote_site_id" json:"remote_site_id"` -} - -type ScrapedSceneMovie struct { - // Set if movie matched - ID *string `graphql:"id" json:"id"` - Name string `graphql:"name" json:"name"` - Aliases string `graphql:"aliases" json:"aliases"` - Duration string `graphql:"duration" json:"duration"` - Date string `graphql:"date" json:"date"` - Rating string `graphql:"rating" json:"rating"` - Director string `graphql:"director" json:"director"` - Synopsis string `graphql:"synopsis" json:"synopsis"` - URL *string `graphql:"url" json:"url"` -} - -type ScrapedSceneTag struct { - // Set if tag matched - ID *string `graphql:"stored_id" json:"stored_id"` - Name string `graphql:"name" json:"name"` -} - -type ScrapedMovie struct { - Name *string `graphql:"name" json:"name"` - Aliases *string `graphql:"aliases" json:"aliases"` - Duration *string `graphql:"duration" json:"duration"` - Date *string `graphql:"date" json:"date"` - Rating *string `graphql:"rating" json:"rating"` - Director *string `graphql:"director" json:"director"` - Studio *ScrapedMovieStudio `graphql:"studio" json:"studio"` - Synopsis *string `graphql:"synopsis" json:"synopsis"` - URL *string `graphql:"url" json:"url"` - FrontImage *string `graphql:"front_image" json:"front_image"` - BackImage *string `graphql:"back_image" json:"back_image"` -} - -type ScrapedMovieStudio struct { - // Set if studio matched - ID *string `graphql:"id" json:"id"` - Name string `graphql:"name" json:"name"` - URL *string `graphql:"url" json:"url"` -} - type ScrapedItems []*ScrapedItem func (s *ScrapedItems) Append(o interface{}) { diff --git a/pkg/models/movie.go b/pkg/models/movie.go index dc6df5fd8..3d11e1e51 100644 --- a/pkg/models/movie.go +++ b/pkg/models/movie.go @@ -11,6 +11,10 @@ type MovieReader interface { Query(movieFilter *MovieFilterType, findFilter *FindFilterType) ([]*Movie, int, error) GetFrontImage(movieID int) ([]byte, error) GetBackImage(movieID int) ([]byte, error) + FindByPerformerID(performerID int) ([]*Movie, error) + CountByPerformerID(performerID int) (int, error) + FindByStudioID(studioID int) ([]*Movie, error) + CountByStudioID(studioID int) (int, error) } type MovieWriter interface { diff --git a/pkg/models/studio.go b/pkg/models/studio.go index 7aa2e87b8..6eec0cdf2 100644 --- a/pkg/models/studio.go +++ b/pkg/models/studio.go @@ -14,6 +14,7 @@ type StudioReader interface { GetImage(studioID int) ([]byte, error) HasImage(studioID int) (bool, error) GetStashIDs(studioID int) ([]*StashID, error) + GetAliases(studioID int) ([]string, error) } type StudioWriter interface { @@ -24,6 +25,7 @@ type StudioWriter interface { UpdateImage(studioID int, image []byte) error DestroyImage(studioID int) error UpdateStashIDs(studioID int, stashIDs []StashID) error + UpdateAliases(studioID int, aliases []string) error } type StudioReaderWriter interface { diff --git a/pkg/models/tag.go b/pkg/models/tag.go index 4d3e0d84e..bcb7096f0 100644 --- a/pkg/models/tag.go +++ b/pkg/models/tag.go @@ -10,6 +10,8 @@ type TagReader interface { FindByGalleryID(galleryID int) ([]*Tag, error) FindByName(name string, nocase bool) (*Tag, error) FindByNames(names []string, nocase bool) ([]*Tag, error) + FindByParentTagID(parentID int) ([]*Tag, error) + FindByChildTagID(childID int) ([]*Tag, error) Count() (int, error) All() ([]*Tag, error) // TODO - this interface is temporary until the filter schema can fully @@ -18,6 +20,8 @@ type TagReader interface { Query(tagFilter *TagFilterType, findFilter *FindFilterType) ([]*Tag, int, error) GetImage(tagID int) ([]byte, error) GetAliases(tagID int) ([]string, error) + FindAllAncestors(tagID int, excludeIDs []int) ([]*Tag, error) + FindAllDescendants(tagID int, excludeIDs []int) ([]*Tag, error) } type TagWriter interface { @@ -29,6 +33,8 @@ type TagWriter interface { DestroyImage(tagID int) error UpdateAliases(tagID int, aliases []string) error Merge(source []int, destination int) error + UpdateParentTags(tagID int, parentIDs []int) error + UpdateChildTags(tagID int, parentIDs []int) error } type TagReaderWriter interface { diff --git a/pkg/models/transaction.go b/pkg/models/transaction.go index 525de5a99..c30f4cb9a 100644 --- a/pkg/models/transaction.go +++ b/pkg/models/transaction.go @@ -1,6 +1,10 @@ package models -import "context" +import ( + "context" + + "github.com/stashapp/stash/pkg/logger" +) type Transaction interface { Begin() error @@ -30,11 +34,15 @@ func WithTxn(txn Transaction, fn func(r Repository) error) error { defer func() { if p := recover(); p != nil { // a panic occurred, rollback and repanic - txn.Rollback() + if err := txn.Rollback(); err != nil { + logger.Warnf("error while trying to roll back transaction: %v", err) + } panic(p) } else if err != nil { // something went wrong, rollback - txn.Rollback() + if err := txn.Rollback(); err != nil { + logger.Warnf("error while trying to roll back transaction: %v", err) + } } else { // all good, commit err = txn.Commit() @@ -54,11 +62,15 @@ func WithROTxn(txn ReadTransaction, fn func(r ReaderRepository) error) error { defer func() { if p := recover(); p != nil { // a panic occurred, rollback and repanic - txn.Rollback() + if err := txn.Rollback(); err != nil { + logger.Warnf("error while trying to roll back RO transaction: %v", err) + } panic(p) } else if err != nil { // something went wrong, rollback - txn.Rollback() + if err := txn.Rollback(); err != nil { + logger.Warnf("error while trying to roll back RO transaction: %v", err) + } } else { // all good, commit err = txn.Commit() diff --git a/pkg/movie/export_test.go b/pkg/movie/export_test.go index b889a0428..b03de426c 100644 --- a/pkg/movie/export_test.go +++ b/pkg/movie/export_test.go @@ -44,18 +44,24 @@ const url = "url" const studioName = "studio" -const frontImage = "ZnJvbnRJbWFnZUJ5dGVz" -const backImage = "YmFja0ltYWdlQnl0ZXM=" +const ( + frontImage = "ZnJvbnRJbWFnZUJ5dGVz" + backImage = "YmFja0ltYWdlQnl0ZXM=" +) -var frontImageBytes = []byte("frontImageBytes") -var backImageBytes = []byte("backImageBytes") +var ( + frontImageBytes = []byte("frontImageBytes") + backImageBytes = []byte("backImageBytes") +) var studio models.Studio = models.Studio{ Name: models.NullString(studioName), } -var createTime time.Time = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) -var updateTime time.Time = time.Date(2002, 01, 01, 0, 0, 0, 0, time.UTC) +var ( + createTime = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) + updateTime = time.Date(2002, 01, 01, 0, 0, 0, 0, time.UTC) +) func createFullMovie(id int, studioID int) models.Movie { return models.Movie{ @@ -142,32 +148,32 @@ var scenarios []testScenario func initTestTable() { scenarios = []testScenario{ - testScenario{ + { createFullMovie(movieID, studioID), createFullJSONMovie(studioName, frontImage, backImage), false, }, - testScenario{ + { createEmptyMovie(emptyID), createEmptyJSONMovie(), false, }, - testScenario{ + { createFullMovie(errFrontImageID, studioID), nil, true, }, - testScenario{ + { createFullMovie(errBackImageID, studioID), nil, true, }, - testScenario{ + { createFullMovie(errStudioMovieID, errStudioID), nil, true, }, - testScenario{ + { createFullMovie(missingStudioMovieID, missingStudioID), createFullJSONMovie("", frontImage, backImage), false, diff --git a/pkg/performer/export_test.go b/pkg/performer/export_test.go index 0d143b2d5..0ec00a93b 100644 --- a/pkg/performer/export_test.go +++ b/pkg/performer/export_test.go @@ -54,8 +54,11 @@ var deathDate = models.SQLiteDate{ String: "2021-02-02", Valid: true, } -var createTime time.Time = time.Date(2001, 01, 01, 0, 0, 0, 0, time.Local) -var updateTime time.Time = time.Date(2002, 01, 01, 0, 0, 0, 0, time.Local) + +var ( + createTime = time.Date(2001, 01, 01, 0, 0, 0, 0, time.Local) + updateTime = time.Date(2002, 01, 01, 0, 0, 0, 0, time.Local) +) func createFullPerformer(id int, name string) *models.Performer { return &models.Performer{ @@ -165,17 +168,17 @@ var scenarios []testScenario func initTestTable() { scenarios = []testScenario{ - testScenario{ + { *createFullPerformer(performerID, performerName), createFullJSONPerformer(performerName, image), false, }, - testScenario{ + { createEmptyPerformer(noImageID), createEmptyJSONPerformer(), false, }, - testScenario{ + { *createFullPerformer(errImageID, performerName), nil, true, diff --git a/pkg/plugin/args.go b/pkg/plugin/args.go index 13fc3570e..adcdf007f 100644 --- a/pkg/plugin/args.go +++ b/pkg/plugin/args.go @@ -17,6 +17,7 @@ func findArg(args []*models.PluginArgInput, name string) *models.PluginArgInput func applyDefaultArgs(args []*models.PluginArgInput, defaultArgs map[string]string) []*models.PluginArgInput { for k, v := range defaultArgs { if arg := findArg(args, k); arg == nil { + v := v // Copy v, because it's being exported out of the loop args = append(args, &models.PluginArgInput{ Key: k, Value: &models.PluginValueInput{ diff --git a/pkg/plugin/common/log/log.go b/pkg/plugin/common/log/log.go index d4ebc0876..e64774a68 100644 --- a/pkg/plugin/common/log/log.go +++ b/pkg/plugin/common/log/log.go @@ -16,144 +16,94 @@ package log import ( - "fmt" "math" - "os" - "strings" + + "github.com/stashapp/stash/pkg/logger" ) // Level represents a logging level for plugin outputs. type Level struct { - char byte - Name string + *logger.PluginLogLevel } // Valid Level values. var ( TraceLevel = Level{ - char: 't', - Name: "trace", + &logger.TraceLevel, } DebugLevel = Level{ - char: 'd', - Name: "debug", + &logger.DebugLevel, } InfoLevel = Level{ - char: 'i', - Name: "info", + &logger.InfoLevel, } WarningLevel = Level{ - char: 'w', - Name: "warning", + &logger.WarningLevel, } ErrorLevel = Level{ - char: 'e', - Name: "error", + &logger.ErrorLevel, } ProgressLevel = Level{ - char: 'p', - Name: "progress", + &logger.ProgressLevel, } NoneLevel = Level{ - Name: "none", + &logger.NoneLevel, } ) -var validLevels = []Level{ - TraceLevel, - DebugLevel, - InfoLevel, - WarningLevel, - ErrorLevel, - ProgressLevel, - NoneLevel, -} - -const startLevelChar byte = 1 -const endLevelChar byte = 2 - -func (l Level) prefix() string { - return string([]byte{ - startLevelChar, - byte(l.char), - endLevelChar, - }) -} - -func (l Level) log(args ...interface{}) { - if l.char == 0 { - return - } - - argsToUse := []interface{}{ - l.prefix(), - } - argsToUse = append(argsToUse, args...) - fmt.Fprintln(os.Stderr, argsToUse...) -} - -func (l Level) logf(format string, args ...interface{}) { - if l.char == 0 { - return - } - - formatToUse := string(l.prefix()) + format + "\n" - fmt.Fprintf(os.Stderr, formatToUse, args...) -} - // Trace outputs a trace logging message to os.Stderr. Message is encoded with a // prefix that signifies to the server that it is a trace message. func Trace(args ...interface{}) { - TraceLevel.log(args...) + TraceLevel.Log(args...) } // Tracef is the equivalent of Printf outputting as a trace logging message. func Tracef(format string, args ...interface{}) { - TraceLevel.logf(format, args...) + TraceLevel.Logf(format, args...) } // Debug outputs a debug logging message to os.Stderr. Message is encoded with a // prefix that signifies to the server that it is a debug message. func Debug(args ...interface{}) { - DebugLevel.log(args...) + DebugLevel.Log(args...) } // Debugf is the equivalent of Printf outputting as a debug logging message. func Debugf(format string, args ...interface{}) { - DebugLevel.logf(format, args...) + DebugLevel.Logf(format, args...) } // Info outputs an info logging message to os.Stderr. Message is encoded with a // prefix that signifies to the server that it is an info message. func Info(args ...interface{}) { - InfoLevel.log(args...) + InfoLevel.Log(args...) } // Infof is the equivalent of Printf outputting as an info logging message. func Infof(format string, args ...interface{}) { - InfoLevel.logf(format, args...) + InfoLevel.Logf(format, args...) } // Warn outputs a warning logging message to os.Stderr. Message is encoded with a // prefix that signifies to the server that it is a warning message. func Warn(args ...interface{}) { - WarningLevel.log(args...) + WarningLevel.Log(args...) } // Warnf is the equivalent of Printf outputting as a warning logging message. func Warnf(format string, args ...interface{}) { - WarningLevel.logf(format, args...) + WarningLevel.Logf(format, args...) } // Error outputs an error logging message to os.Stderr. Message is encoded with a // prefix that signifies to the server that it is an error message. func Error(args ...interface{}) { - ErrorLevel.log(args...) + ErrorLevel.Log(args...) } // Errorf is the equivalent of Printf outputting as an error logging message. func Errorf(format string, args ...interface{}) { - ErrorLevel.logf(format, args...) + ErrorLevel.Logf(format, args...) } // Progress logs the current progress value. The progress value should be @@ -161,15 +111,16 @@ func Errorf(format string, args ...interface{}) { // complete. Values outside of this range will be clamp to be within it. func Progress(progress float64) { progress = math.Min(math.Max(0, progress), 1) - ProgressLevel.log(progress) + ProgressLevel.Log(progress) } // LevelFromName returns the Level that matches the provided name or nil if // the name does not match a valid value. func LevelFromName(name string) *Level { - for _, l := range validLevels { - if l.Name == name { - return &l + l := logger.PluginLogLevelFromName(name) + if l != nil { + return &Level{ + l, } } @@ -181,24 +132,14 @@ func LevelFromName(name string) *Level { // determines the log level, if present. If not present, the plugin output // is returned unchanged with a nil Level. func DetectLogLevel(line string) (*Level, string) { - if len(line) < 4 || line[0] != startLevelChar || line[2] != endLevelChar { - return nil, line - } - - char := line[1] - var level *Level - for _, l := range validLevels { - if l.char == char { - level = &l - break - } - } + var level *logger.PluginLogLevel + level, line = logger.DetectLogLevel(line) if level == nil { return nil, line } - line = strings.TrimSpace(line[3:]) - - return level, line + return &Level{ + level, + }, line } diff --git a/pkg/plugin/examples/common/graphql.go b/pkg/plugin/examples/common/graphql.go index 3ed5ec341..624c0d5d8 100644 --- a/pkg/plugin/examples/common/graphql.go +++ b/pkg/plugin/examples/common/graphql.go @@ -1,3 +1,4 @@ +//go:build plugin_example // +build plugin_example package common diff --git a/pkg/plugin/examples/goraw/main.go b/pkg/plugin/examples/goraw/main.go index 26060f32c..c3b287bb4 100644 --- a/pkg/plugin/examples/goraw/main.go +++ b/pkg/plugin/examples/goraw/main.go @@ -1,10 +1,11 @@ +//go:build plugin_example // +build plugin_example package main import ( "encoding/json" - "io/ioutil" + "io" "os" "time" @@ -22,7 +23,7 @@ func main() { input := common.PluginInput{} if len(os.Args) < 2 { - inData, _ := ioutil.ReadAll(os.Stdin) + inData, _ := io.ReadAll(os.Stdin) log.Debugf("Raw input: %s", string(inData)) decodeErr := json.Unmarshal(inData, &input) diff --git a/pkg/plugin/examples/gorpc/main.go b/pkg/plugin/examples/gorpc/main.go index 75e364549..1a86131cc 100644 --- a/pkg/plugin/examples/gorpc/main.go +++ b/pkg/plugin/examples/gorpc/main.go @@ -1,3 +1,4 @@ +//go:build plugin_example // +build plugin_example package main diff --git a/pkg/plugin/js.go b/pkg/plugin/js.go index 6b070ce46..1c8f18a32 100644 --- a/pkg/plugin/js.go +++ b/pkg/plugin/js.go @@ -2,6 +2,7 @@ package plugin import ( "errors" + "fmt" "path/filepath" "sync" @@ -71,10 +72,21 @@ func (t *jsPluginTask) Start() error { return err } - t.vm.Set("input", t.input) - js.AddLogAPI(t.vm, t.progress) - js.AddUtilAPI(t.vm) - js.AddGQLAPI(t.vm, t.input.ServerConnection.SessionCookie, t.gqlHandler) + if err := t.vm.Set("input", t.input); err != nil { + return fmt.Errorf("error setting input: %w", err) + } + + if err := js.AddLogAPI(t.vm, t.progress); err != nil { + return fmt.Errorf("error adding log API: %w", err) + } + + if err := js.AddUtilAPI(t.vm); err != nil { + return fmt.Errorf("error adding util API: %w", err) + } + + if err := js.AddGQLAPI(t.vm, t.input.ServerConnection.SessionCookie, t.gqlHandler); err != nil { + return fmt.Errorf("error adding GraphQL API: %w", err) + } t.vm.Interrupt = make(chan func(), 1) diff --git a/pkg/plugin/js/gql.go b/pkg/plugin/js/gql.go index 13d5fe003..45f9ac9e7 100644 --- a/pkg/plugin/js/gql.go +++ b/pkg/plugin/js/gql.go @@ -103,9 +103,15 @@ func gqlRequestFunc(vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) } } -func AddGQLAPI(vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) { +func AddGQLAPI(vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) error { gql, _ := vm.Object("({})") - gql.Set("Do", gqlRequestFunc(vm, cookie, gqlHandler)) + if err := gql.Set("Do", gqlRequestFunc(vm, cookie, gqlHandler)); err != nil { + return fmt.Errorf("unable to set GraphQL Do function: %w", err) + } - vm.Set("gql", gql) + if err := vm.Set("gql", gql); err != nil { + return fmt.Errorf("unable to set gql: %w", err) + } + + return nil } diff --git a/pkg/plugin/js/log.go b/pkg/plugin/js/log.go index 35d23a537..9f5acde89 100644 --- a/pkg/plugin/js/log.go +++ b/pkg/plugin/js/log.go @@ -2,6 +2,7 @@ package js import ( "encoding/json" + "fmt" "math" "github.com/robertkrimen/otto" @@ -64,14 +65,29 @@ func logProgressFunc(c chan float64) func(call otto.FunctionCall) otto.Value { } } -func AddLogAPI(vm *otto.Otto, progress chan float64) { +func AddLogAPI(vm *otto.Otto, progress chan float64) error { log, _ := vm.Object("({})") - log.Set("Trace", logTrace) - log.Set("Debug", logDebug) - log.Set("Info", logInfo) - log.Set("Warn", logWarn) - log.Set("Error", logError) - log.Set("Progress", logProgressFunc(progress)) + if err := log.Set("Trace", logTrace); err != nil { + return fmt.Errorf("error setting Trace: %w", err) + } + if err := log.Set("Debug", logDebug); err != nil { + return fmt.Errorf("error setting Debug: %w", err) + } + if err := log.Set("Info", logInfo); err != nil { + return fmt.Errorf("error setting Info: %w", err) + } + if err := log.Set("Warn", logWarn); err != nil { + return fmt.Errorf("error setting Warn: %w", err) + } + if err := log.Set("Error", logError); err != nil { + return fmt.Errorf("error setting Error: %w", err) + } + if err := log.Set("Progress", logProgressFunc(progress)); err != nil { + return fmt.Errorf("error setting Progress: %v", err) + } + if err := vm.Set("log", log); err != nil { + return fmt.Errorf("unable to set log: %w", err) + } - vm.Set("log", log) + return nil } diff --git a/pkg/plugin/js/util.go b/pkg/plugin/js/util.go index 4590d5e25..1fe5bb3ff 100644 --- a/pkg/plugin/js/util.go +++ b/pkg/plugin/js/util.go @@ -1,6 +1,7 @@ package js import ( + "fmt" "time" "github.com/robertkrimen/otto" @@ -14,9 +15,15 @@ func sleepFunc(call otto.FunctionCall) otto.Value { return otto.UndefinedValue() } -func AddUtilAPI(vm *otto.Otto) { +func AddUtilAPI(vm *otto.Otto) error { util, _ := vm.Object("({})") - util.Set("Sleep", sleepFunc) + if err := util.Set("Sleep", sleepFunc); err != nil { + return fmt.Errorf("unable to set sleep func: %w", err) + } - vm.Set("util", util) + if err := vm.Set("util", util); err != nil { + return fmt.Errorf("unable to set util: %w", err) + } + + return nil } diff --git a/pkg/plugin/log.go b/pkg/plugin/log.go index 96727f688..0b8511521 100644 --- a/pkg/plugin/log.go +++ b/pkg/plugin/log.go @@ -1,79 +1,26 @@ package plugin import ( - "bufio" + "fmt" "io" - "strconv" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/plugin/common/log" ) -func (t *pluginTask) handleStderrLine(line string, defaultLogLevel *log.Level) { - level, l := log.DetectLogLevel(line) - - const pluginPrefix = "[Plugin] " - // if no log level, just output to info - if level == nil { - if defaultLogLevel != nil { - level = defaultLogLevel - } else { - level = &log.InfoLevel - } - } - - switch *level { - case log.TraceLevel: - logger.Trace(pluginPrefix, l) - case log.DebugLevel: - logger.Debug(pluginPrefix, l) - case log.InfoLevel: - logger.Info(pluginPrefix, l) - case log.WarningLevel: - logger.Warn(pluginPrefix, l) - case log.ErrorLevel: - logger.Error(pluginPrefix, l) - case log.ProgressLevel: - progress, err := strconv.ParseFloat(l, 64) - if err != nil { - logger.Errorf("Error parsing progress value '%s': %s", l, err.Error()) - } else { - // only pass progress through if channel present - if t.progress != nil { - // don't block on this - select { - case t.progress <- progress: - default: - } - } - } - } -} - -func (t *pluginTask) handlePluginOutput(pluginOutputReader io.ReadCloser, defaultLogLevel *log.Level) { - // pipe plugin stderr to our logging - scanner := bufio.NewScanner(pluginOutputReader) - for scanner.Scan() { - str := scanner.Text() - if str != "" { - t.handleStderrLine(str, defaultLogLevel) - } - } - - str := scanner.Text() - if str != "" { - t.handleStderrLine(str, defaultLogLevel) - } - - pluginOutputReader.Close() -} - -func (t *pluginTask) handlePluginStderr(pluginOutputReader io.ReadCloser) { - logLevel := log.LevelFromName(t.plugin.PluginErrLogLevel) +func (t *pluginTask) handlePluginStderr(name string, pluginOutputReader io.ReadCloser) { + logLevel := logger.PluginLogLevelFromName(t.plugin.PluginErrLogLevel) if logLevel == nil { // default log level to error - logLevel = &log.ErrorLevel + logLevel = &logger.ErrorLevel } - t.handlePluginOutput(pluginOutputReader, logLevel) + const pluginPrefix = "[Plugin / %s] " + + lgr := logger.PluginLogger{ + Prefix: fmt.Sprintf(pluginPrefix, name), + DefaultLogLevel: logLevel, + ProgressChan: t.progress, + } + + lgr.HandlePluginStdErr(pluginOutputReader) } diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 844834339..7322221a9 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -219,7 +219,9 @@ func (c Cache) executePostHooks(ctx context.Context, hookType HookTriggerEnum, h select { case <-ctx.Done(): - task.Stop() + if err := task.Stop(); err != nil { + logger.Warnf("could not stop task: %v", err) + } return fmt.Errorf("operation cancelled") case <-c: // task finished normally diff --git a/pkg/plugin/raw.go b/pkg/plugin/raw.go index fe44c2f6d..03d084ccd 100644 --- a/pkg/plugin/raw.go +++ b/pkg/plugin/raw.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os/exec" "sync" @@ -51,33 +50,35 @@ func (t *rawPluginTask) Start() error { defer stdin.Close() inBytes, _ := json.Marshal(t.input) - io.WriteString(stdin, string(inBytes)) + if k, err := io.WriteString(stdin, string(inBytes)); err != nil { + logger.Warnf("error writing input to plugins stdin (wrote %v bytes out of %v): %v", k, len(string(inBytes)), err) + } }() stderr, err := cmd.StderrPipe() if err != nil { - logger.Error("Plugin stderr not available: " + err.Error()) + logger.Error("plugin stderr not available: " + err.Error()) } stdout, err := cmd.StdoutPipe() if nil != err { - logger.Error("Plugin stdout not available: " + err.Error()) + logger.Error("plugin stdout not available: " + err.Error()) } t.waitGroup.Add(1) t.done = make(chan bool, 1) if err = cmd.Start(); err != nil { - return fmt.Errorf("Error running plugin: %s", err.Error()) + return fmt.Errorf("error running plugin: %s", err.Error()) } - go t.handlePluginStderr(stderr) + go t.handlePluginStderr(t.plugin.Name, stderr) t.cmd = cmd // send the stdout to the plugin output go func() { defer t.waitGroup.Done() defer close(t.done) - stdoutData, _ := ioutil.ReadAll(stdout) + stdoutData, _ := io.ReadAll(stdout) stdoutString := string(stdoutData) output := t.getOutput(stdoutString) diff --git a/pkg/plugin/rpc.go b/pkg/plugin/rpc.go index dff9774c0..49955a55b 100644 --- a/pkg/plugin/rpc.go +++ b/pkg/plugin/rpc.go @@ -64,7 +64,7 @@ func (t *rpcPluginTask) Start() error { return err } - go t.handlePluginStderr(pluginErrReader) + go t.handlePluginStderr(t.plugin.Name, pluginErrReader) iface := rpcPluginClient{ Client: t.client, diff --git a/pkg/scene/export_test.go b/pkg/scene/export_test.go index dc3164f13..b0b4f7834 100644 --- a/pkg/scene/export_test.go +++ b/pkg/scene/export_test.go @@ -23,8 +23,8 @@ const ( missingStudioID = 5 errStudioID = 6 - noGalleryID = 7 - errGalleryID = 8 + // noGalleryID = 7 + // errGalleryID = 8 noTagsID = 11 errTagsID = 12 @@ -64,8 +64,8 @@ const ( ) const ( - studioName = "studioName" - galleryChecksum = "galleryChecksum" + studioName = "studioName" + // galleryChecksum = "galleryChecksum" validMovie1 = 1 validMovie2 = 2 @@ -87,8 +87,10 @@ var imageBytes = []byte("imageBytes") const image = "aW1hZ2VCeXRlcw==" -var createTime time.Time = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) -var updateTime time.Time = time.Date(2002, 01, 01, 0, 0, 0, 0, time.UTC) +var ( + createTime = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) + updateTime = time.Date(2002, 01, 01, 0, 0, 0, 0, time.UTC) +) func createFullScene(id int) models.Scene { return models.Scene{ @@ -293,24 +295,6 @@ func TestGetStudioName(t *testing.T) { mockStudioReader.AssertExpectations(t) } -var getGalleryChecksumScenarios = []stringTestScenario{ - { - createEmptyScene(sceneID), - galleryChecksum, - false, - }, - { - createEmptyScene(noGalleryID), - "", - false, - }, - { - createEmptyScene(errGalleryID), - "", - true, - }, -} - type stringSliceTestScenario struct { input models.Scene expected []string diff --git a/pkg/scene/import_test.go b/pkg/scene/import_test.go index 09f0bf38d..3f59c3cf3 100644 --- a/pkg/scene/import_test.go +++ b/pkg/scene/import_test.go @@ -16,8 +16,8 @@ const invalidImage = "aW1hZ2VCeXRlcw&&" const ( path = "path" - sceneNameErr = "sceneNameErr" - existingSceneName = "existingSceneName" + sceneNameErr = "sceneNameErr" + // existingSceneName = "existingSceneName" existingSceneID = 100 existingStudioID = 101 diff --git a/pkg/scraper/action.go b/pkg/scraper/action.go index ca7e82b2c..493163936 100644 --- a/pkg/scraper/action.go +++ b/pkg/scraper/action.go @@ -11,13 +11,6 @@ const ( scraperActionJson scraperAction = "scrapeJson" ) -var allScraperAction = []scraperAction{ - scraperActionScript, - scraperActionStash, - scraperActionXPath, - scraperActionJson, -} - func (e scraperAction) IsValid() bool { switch e { case scraperActionScript, scraperActionStash, scraperActionXPath, scraperActionJson: @@ -26,21 +19,18 @@ func (e scraperAction) IsValid() bool { return false } -type scrapeOptions struct { - scraper scraperTypeConfig - config config - globalConfig GlobalConfig -} - type scraper interface { scrapePerformersByName(name string) ([]*models.ScrapedPerformer, error) scrapePerformerByFragment(scrapedPerformer models.ScrapedPerformerInput) (*models.ScrapedPerformer, error) scrapePerformerByURL(url string) (*models.ScrapedPerformer, error) - scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) + scrapeScenesByName(name string) ([]*models.ScrapedScene, error) + scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) + scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) scrapeSceneByURL(url string) (*models.ScrapedScene, error) - scrapeGalleryByFragment(scene models.GalleryUpdateInput) (*models.ScrapedGallery, error) + scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) + scrapeGalleryByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) scrapeGalleryByURL(url string) (*models.ScrapedGallery, error) scrapeMovieByURL(url string) (*models.ScrapedMovie, error) diff --git a/pkg/scraper/config.go b/pkg/scraper/config.go index c16d55a7f..78d3fe4fe 100644 --- a/pkg/scraper/config.go +++ b/pkg/scraper/config.go @@ -35,6 +35,12 @@ type config struct { // Configuration for querying gallery by a Gallery fragment GalleryByFragment *scraperTypeConfig `yaml:"galleryByFragment"` + // Configuration for querying scenes by name + SceneByName *scraperTypeConfig `yaml:"sceneByName"` + + // Configuration for querying scenes by query fragment + SceneByQueryFragment *scraperTypeConfig `yaml:"sceneByQueryFragment"` + // Configuration for querying a scene by a URL SceneByURL []*scrapeByURLConfig `yaml:"sceneByURL"` @@ -256,6 +262,9 @@ func (c config) toScraper() *models.Scraper { if c.SceneByFragment != nil { scene.SupportedScrapes = append(scene.SupportedScrapes, models.ScrapeTypeFragment) } + if c.SceneByName != nil && c.SceneByQueryFragment != nil { + scene.SupportedScrapes = append(scene.SupportedScrapes, models.ScrapeTypeName) + } if len(c.SceneByURL) > 0 { scene.SupportedScrapes = append(scene.SupportedScrapes, models.ScrapeTypeURL) for _, v := range c.SceneByURL { @@ -353,7 +362,7 @@ func (c config) ScrapePerformerURL(url string, txnManager models.TransactionMana } func (c config) supportsScenes() bool { - return c.SceneByFragment != nil || len(c.SceneByURL) > 0 + return (c.SceneByName != nil && c.SceneByQueryFragment != nil) || c.SceneByFragment != nil || len(c.SceneByURL) > 0 } func (c config) supportsGalleries() bool { @@ -393,9 +402,27 @@ func (c config) matchesMovieURL(url string) bool { return false } -func (c config) ScrapeScene(scene models.SceneUpdateInput, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedScene, error) { +func (c config) ScrapeSceneQuery(name string, txnManager models.TransactionManager, globalConfig GlobalConfig) ([]*models.ScrapedScene, error) { + if c.SceneByName != nil { + s := getScraper(*c.SceneByName, txnManager, c, globalConfig) + return s.scrapeScenesByName(name) + } + + return nil, nil +} + +func (c config) ScrapeSceneByScene(scene *models.Scene, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedScene, error) { if c.SceneByFragment != nil { s := getScraper(*c.SceneByFragment, txnManager, c, globalConfig) + return s.scrapeSceneByScene(scene) + } + + return nil, nil +} + +func (c config) ScrapeSceneByFragment(scene models.ScrapedSceneInput, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedScene, error) { + if c.SceneByQueryFragment != nil { + s := getScraper(*c.SceneByQueryFragment, txnManager, c, globalConfig) return s.scrapeSceneByFragment(scene) } @@ -420,8 +447,18 @@ func (c config) ScrapeSceneURL(url string, txnManager models.TransactionManager, return nil, nil } -func (c config) ScrapeGallery(gallery models.GalleryUpdateInput, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedGallery, error) { +func (c config) ScrapeGalleryByGallery(gallery *models.Gallery, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedGallery, error) { if c.GalleryByFragment != nil { + s := getScraper(*c.GalleryByFragment, txnManager, c, globalConfig) + return s.scrapeGalleryByGallery(gallery) + } + + return nil, nil +} + +func (c config) ScrapeGalleryByFragment(gallery models.ScrapedGalleryInput, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedGallery, error) { + if c.GalleryByFragment != nil { + // TODO - this should be galleryByQueryFragment s := getScraper(*c.GalleryByFragment, txnManager, c, globalConfig) return s.scrapeGalleryByFragment(gallery) } diff --git a/pkg/scraper/image.go b/pkg/scraper/image.go index ab09f28da..e2b3b5e1e 100644 --- a/pkg/scraper/image.go +++ b/pkg/scraper/image.go @@ -3,7 +3,7 @@ package scraper import ( "crypto/tls" "fmt" - "io/ioutil" + "io" "net/http" "strings" "time" @@ -28,6 +28,8 @@ func setPerformerImage(p *models.ScrapedPerformer, globalConfig GlobalConfig) er } p.Image = img + // Image is deprecated. Use images instead + p.Images = []string{*img} return nil } @@ -119,7 +121,7 @@ func getImage(url string, globalConfig GlobalConfig) (*string, error) { defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/pkg/scraper/json.go b/pkg/scraper/json.go index b7c68e86e..e99a4b698 100644 --- a/pkg/scraper/json.go +++ b/pkg/scraper/json.go @@ -2,7 +2,7 @@ package scraper import ( "errors" - "io/ioutil" + "io" "net/url" "strings" @@ -53,7 +53,7 @@ func (s *jsonScraper) loadURL(url string) (string, error) { return "", err } logger.Infof("loadURL (%s)\n", url) - doc, err := ioutil.ReadAll(r) + doc, err := io.ReadAll(r) if err != nil { return "", err } @@ -143,18 +143,34 @@ func (s *jsonScraper) scrapePerformerByFragment(scrapedPerformer models.ScrapedP return nil, errors.New("scrapePerformerByFragment not supported for json scraper") } -func (s *jsonScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) { - storedScene, err := sceneFromUpdateFragment(scene, s.txnManager) +func (s *jsonScraper) scrapeScenesByName(name string) ([]*models.ScrapedScene, error) { + scraper := s.getJsonScraper() + + if scraper == nil { + return nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config") + } + + const placeholder = "{}" + + // replace the placeholder string with the URL-escaped name + escapedName := url.QueryEscape(name) + + url := s.scraper.QueryURL + url = strings.Replace(url, placeholder, escapedName, -1) + + doc, err := s.loadURL(url) + if err != nil { return nil, err } - if storedScene == nil { - return nil, errors.New("no scene found") - } + q := s.getJsonQuery(doc) + return scraper.scrapeScenes(q) +} +func (s *jsonScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) { // construct the URL - queryURL := queryURLParametersFromScene(storedScene) + queryURL := queryURLParametersFromScene(scene) if s.scraper.QueryURLReplacements != nil { queryURL.applyReplacements(s.scraper.QueryURLReplacements) } @@ -176,18 +192,33 @@ func (s *jsonScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mod return scraper.scrapeScene(q) } -func (s *jsonScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) { - storedGallery, err := galleryFromUpdateFragment(gallery, s.txnManager) +func (s *jsonScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) { + // construct the URL + queryURL := queryURLParametersFromScrapedScene(scene) + if s.scraper.QueryURLReplacements != nil { + queryURL.applyReplacements(s.scraper.QueryURLReplacements) + } + url := queryURL.constructURL(s.scraper.QueryURL) + + scraper := s.getJsonScraper() + + if scraper == nil { + return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config") + } + + doc, err := s.loadURL(url) + if err != nil { return nil, err } - if storedGallery == nil { - return nil, errors.New("no scene found") - } + q := s.getJsonQuery(doc) + return scraper.scrapeScene(q) +} +func (s *jsonScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) { // construct the URL - queryURL := queryURLParametersFromGallery(storedGallery) + queryURL := queryURLParametersFromGallery(gallery) if s.scraper.QueryURLReplacements != nil { queryURL.applyReplacements(s.scraper.QueryURLReplacements) } @@ -209,6 +240,10 @@ func (s *jsonScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) return scraper.scrapeGallery(q) } +func (s *jsonScraper) scrapeGalleryByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) { + return nil, errors.New("scrapeGalleryByFragment not supported for json scraper") +} + func (s *jsonScraper) getJsonQuery(doc string) *jsonQuery { return &jsonQuery{ doc: doc, diff --git a/pkg/scraper/mapped.go b/pkg/scraper/mapped.go index 5cbdead74..80d854341 100644 --- a/pkg/scraper/mapped.go +++ b/pkg/scraper/mapped.go @@ -32,9 +32,7 @@ func (s mappedConfig) applyCommon(c commonMappedConfig, src string) string { ret := src for commonKey, commonVal := range c { - if strings.Contains(ret, commonKey) { - ret = strings.Replace(ret, commonKey, commonVal, -1) - } + ret = strings.Replace(ret, commonKey, commonVal, -1) } return ret @@ -557,7 +555,7 @@ func (a mappedPostProcessAction) ToPostProcessAction() (postProcessAction, error if found != "" { return nil, fmt.Errorf("post-process actions must have a single field, found %s and %s", found, "subtractDays") } - found = "subtractDays" + // found = "subtractDays" action := postProcessSubtractDays(a.SubtractDays) ret = &action } @@ -763,7 +761,7 @@ func (s mappedScraper) scrapePerformer(q mappedQuery) (*models.ScrapedPerformer, tagResults := performerTagsMap.process(q, s.Common) for _, p := range tagResults { - tag := &models.ScrapedSceneTag{} + tag := &models.ScrapedTag{} p.apply(tag) ret.Tags = append(ret.Tags, tag) } @@ -791,6 +789,100 @@ func (s mappedScraper) scrapePerformers(q mappedQuery) ([]*models.ScrapedPerform return ret, nil } +func (s mappedScraper) processScene(q mappedQuery, r mappedResult) *models.ScrapedScene { + var ret models.ScrapedScene + + sceneScraperConfig := s.Scene + + scenePerformersMap := sceneScraperConfig.Performers + sceneTagsMap := sceneScraperConfig.Tags + sceneStudioMap := sceneScraperConfig.Studio + sceneMoviesMap := sceneScraperConfig.Movies + + scenePerformerTagsMap := scenePerformersMap.Tags + + r.apply(&ret) + + // process performer tags once + var performerTagResults mappedResults + if scenePerformerTagsMap != nil { + performerTagResults = scenePerformerTagsMap.process(q, s.Common) + } + + // now apply the performers and tags + if scenePerformersMap.mappedConfig != nil { + logger.Debug(`Processing scene performers:`) + performerResults := scenePerformersMap.process(q, s.Common) + + for _, p := range performerResults { + performer := &models.ScrapedPerformer{} + p.apply(performer) + + for _, p := range performerTagResults { + tag := &models.ScrapedTag{} + p.apply(tag) + ret.Tags = append(ret.Tags, tag) + } + + ret.Performers = append(ret.Performers, performer) + } + } + + if sceneTagsMap != nil { + logger.Debug(`Processing scene tags:`) + tagResults := sceneTagsMap.process(q, s.Common) + + for _, p := range tagResults { + tag := &models.ScrapedTag{} + p.apply(tag) + ret.Tags = append(ret.Tags, tag) + } + } + + if sceneStudioMap != nil { + logger.Debug(`Processing scene studio:`) + studioResults := sceneStudioMap.process(q, s.Common) + + if len(studioResults) > 0 { + studio := &models.ScrapedStudio{} + studioResults[0].apply(studio) + ret.Studio = studio + } + } + + if sceneMoviesMap != nil { + logger.Debug(`Processing scene movies:`) + movieResults := sceneMoviesMap.process(q, s.Common) + + for _, p := range movieResults { + movie := &models.ScrapedMovie{} + p.apply(movie) + ret.Movies = append(ret.Movies, movie) + } + } + + return &ret +} + +func (s mappedScraper) scrapeScenes(q mappedQuery) ([]*models.ScrapedScene, error) { + var ret []*models.ScrapedScene + + sceneScraperConfig := s.Scene + sceneMap := sceneScraperConfig.mappedConfig + if sceneMap == nil { + return nil, nil + } + + logger.Debug(`Processing scenes:`) + results := sceneMap.process(q, s.Common) + for _, r := range results { + logger.Debug(`Processing scene:`) + ret = append(ret, s.processScene(q, r)) + } + + return ret, nil +} + func (s mappedScraper) scrapeScene(q mappedQuery) (*models.ScrapedScene, error) { var ret models.ScrapedScene @@ -800,76 +892,11 @@ func (s mappedScraper) scrapeScene(q mappedQuery) (*models.ScrapedScene, error) return nil, nil } - scenePerformersMap := sceneScraperConfig.Performers - sceneTagsMap := sceneScraperConfig.Tags - sceneStudioMap := sceneScraperConfig.Studio - sceneMoviesMap := sceneScraperConfig.Movies - - scenePerformerTagsMap := scenePerformersMap.Tags - logger.Debug(`Processing scene:`) results := sceneMap.process(q, s.Common) if len(results) > 0 { - results[0].apply(&ret) - - // process performer tags once - var performerTagResults mappedResults - if scenePerformerTagsMap != nil { - performerTagResults = scenePerformerTagsMap.process(q, s.Common) - } - - // now apply the performers and tags - if scenePerformersMap.mappedConfig != nil { - logger.Debug(`Processing scene performers:`) - performerResults := scenePerformersMap.process(q, s.Common) - - for _, p := range performerResults { - performer := &models.ScrapedScenePerformer{} - p.apply(performer) - - for _, p := range performerTagResults { - tag := &models.ScrapedSceneTag{} - p.apply(tag) - ret.Tags = append(ret.Tags, tag) - } - - ret.Performers = append(ret.Performers, performer) - } - } - - if sceneTagsMap != nil { - logger.Debug(`Processing scene tags:`) - tagResults := sceneTagsMap.process(q, s.Common) - - for _, p := range tagResults { - tag := &models.ScrapedSceneTag{} - p.apply(tag) - ret.Tags = append(ret.Tags, tag) - } - } - - if sceneStudioMap != nil { - logger.Debug(`Processing scene studio:`) - studioResults := sceneStudioMap.process(q, s.Common) - - if len(studioResults) > 0 { - studio := &models.ScrapedSceneStudio{} - studioResults[0].apply(studio) - ret.Studio = studio - } - } - - if sceneMoviesMap != nil { - logger.Debug(`Processing scene movies:`) - movieResults := sceneMoviesMap.process(q, s.Common) - - for _, p := range movieResults { - movie := &models.ScrapedSceneMovie{} - p.apply(movie) - ret.Movies = append(ret.Movies, movie) - } - - } + ss := s.processScene(q, results[0]) + ret = *ss } return &ret, nil @@ -899,7 +926,7 @@ func (s mappedScraper) scrapeGallery(q mappedQuery) (*models.ScrapedGallery, err performerResults := galleryPerformersMap.process(q, s.Common) for _, p := range performerResults { - performer := &models.ScrapedScenePerformer{} + performer := &models.ScrapedPerformer{} p.apply(performer) ret.Performers = append(ret.Performers, performer) } @@ -910,7 +937,7 @@ func (s mappedScraper) scrapeGallery(q mappedQuery) (*models.ScrapedGallery, err tagResults := galleryTagsMap.process(q, s.Common) for _, p := range tagResults { - tag := &models.ScrapedSceneTag{} + tag := &models.ScrapedTag{} p.apply(tag) ret.Tags = append(ret.Tags, tag) } @@ -921,7 +948,7 @@ func (s mappedScraper) scrapeGallery(q mappedQuery) (*models.ScrapedGallery, err studioResults := galleryStudioMap.process(q, s.Common) if len(studioResults) > 0 { - studio := &models.ScrapedSceneStudio{} + studio := &models.ScrapedStudio{} studioResults[0].apply(studio) ret.Studio = studio } @@ -951,7 +978,7 @@ func (s mappedScraper) scrapeMovie(q mappedQuery) (*models.ScrapedMovie, error) studioResults := movieStudioMap.process(q, s.Common) if len(studioResults) > 0 { - studio := &models.ScrapedMovieStudio{} + studio := &models.ScrapedStudio{} studioResults[0].apply(studio) ret.Studio = studio } diff --git a/pkg/scraper/matchers.go b/pkg/scraper/matchers.go index cc1d6f99c..f129ec8b8 100644 --- a/pkg/scraper/matchers.go +++ b/pkg/scraper/matchers.go @@ -4,13 +4,18 @@ import ( "strconv" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/studio" "github.com/stashapp/stash/pkg/tag" ) -// MatchScrapedScenePerformer matches the provided performer with the +// MatchScrapedPerformer matches the provided performer with the // performers in the database and sets the ID field if one is found. -func MatchScrapedScenePerformer(qb models.PerformerReader, p *models.ScrapedScenePerformer) error { - performers, err := qb.FindByNames([]string{p.Name}, true) +func MatchScrapedPerformer(qb models.PerformerReader, p *models.ScrapedPerformer) error { + if p.Name == nil { + return nil + } + + performers, err := qb.FindByNames([]string{*p.Name}, true) if err != nil { return err @@ -22,33 +27,45 @@ func MatchScrapedScenePerformer(qb models.PerformerReader, p *models.ScrapedScen } id := strconv.Itoa(performers[0].ID) - p.ID = &id + p.StoredID = &id return nil } -// MatchScrapedSceneStudio matches the provided studio with the studios +// MatchScrapedStudio matches the provided studio with the studios // in the database and sets the ID field if one is found. -func MatchScrapedSceneStudio(qb models.StudioReader, s *models.ScrapedSceneStudio) error { - studio, err := qb.FindByName(s.Name, true) +func MatchScrapedStudio(qb models.StudioReader, s *models.ScrapedStudio) error { + st, err := studio.ByName(qb, s.Name) if err != nil { return err } - if studio == nil { + if st == nil { + // try matching by alias + st, err = studio.ByAlias(qb, s.Name) + if err != nil { + return err + } + } + + if st == nil { // ignore - cannot match return nil } - id := strconv.Itoa(studio.ID) - s.ID = &id + id := strconv.Itoa(st.ID) + s.StoredID = &id return nil } -// MatchScrapedSceneMovie matches the provided movie with the movies +// MatchScrapedMovie matches the provided movie with the movies // in the database and sets the ID field if one is found. -func MatchScrapedSceneMovie(qb models.MovieReader, m *models.ScrapedSceneMovie) error { - movies, err := qb.FindByNames([]string{m.Name}, true) +func MatchScrapedMovie(qb models.MovieReader, m *models.ScrapedMovie) error { + if m.Name == nil { + return nil + } + + movies, err := qb.FindByNames([]string{*m.Name}, true) if err != nil { return err @@ -60,13 +77,13 @@ func MatchScrapedSceneMovie(qb models.MovieReader, m *models.ScrapedSceneMovie) } id := strconv.Itoa(movies[0].ID) - m.ID = &id + m.StoredID = &id return nil } -// MatchScrapedSceneTag matches the provided tag with the tags +// MatchScrapedTag matches the provided tag with the tags // in the database and sets the ID field if one is found. -func MatchScrapedSceneTag(qb models.TagReader, s *models.ScrapedSceneTag) error { +func MatchScrapedTag(qb models.TagReader, s *models.ScrapedTag) error { t, err := tag.ByName(qb, s.Name) if err != nil { @@ -87,6 +104,6 @@ func MatchScrapedSceneTag(qb models.TagReader, s *models.ScrapedSceneTag) error } id := strconv.Itoa(t.ID) - s.ID = &id + s.StoredID = &id return nil } diff --git a/pkg/scraper/query_url.go b/pkg/scraper/query_url.go index 462069d2f..b48b2b794 100644 --- a/pkg/scraper/query_url.go +++ b/pkg/scraper/query_url.go @@ -21,6 +21,23 @@ func queryURLParametersFromScene(scene *models.Scene) queryURLParameters { return ret } +func queryURLParametersFromScrapedScene(scene models.ScrapedSceneInput) queryURLParameters { + ret := make(queryURLParameters) + + setField := func(field string, value *string) { + if value != nil { + ret[field] = *value + } + } + + setField("title", scene.Title) + setField("url", scene.URL) + setField("date", scene.Date) + setField("details", scene.Details) + setField("remote_site_id", scene.RemoteSiteID) + return ret +} + func queryURLParameterFromURL(url string) queryURLParameters { ret := make(queryURLParameters) ret["url"] = url diff --git a/pkg/scraper/scrapers.go b/pkg/scraper/scrapers.go index 6c8d6e09d..d039a59a7 100644 --- a/pkg/scraper/scrapers.go +++ b/pkg/scraper/scrapers.go @@ -6,7 +6,6 @@ import ( "os" "path/filepath" "regexp" - "strconv" "strings" "github.com/stashapp/stash/pkg/logger" @@ -260,7 +259,7 @@ func (c Cache) postScrapePerformer(ret *models.ScrapedPerformer) error { return nil } -func (c Cache) postScrapeScenePerformer(ret *models.ScrapedScenePerformer) error { +func (c Cache) postScrapeScenePerformer(ret *models.ScrapedPerformer) error { if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { tqb := r.Tag() @@ -290,13 +289,13 @@ func (c Cache) postScrapeScene(ret *models.ScrapedScene) error { return err } - if err := MatchScrapedScenePerformer(pqb, p); err != nil { + if err := MatchScrapedPerformer(pqb, p); err != nil { return err } } for _, p := range ret.Movies { - err := MatchScrapedSceneMovie(mqb, p) + err := MatchScrapedMovie(mqb, p) if err != nil { return err } @@ -309,7 +308,7 @@ func (c Cache) postScrapeScene(ret *models.ScrapedScene) error { ret.Tags = tags if ret.Studio != nil { - err := MatchScrapedSceneStudio(sqb, ret.Studio) + err := MatchScrapedStudio(sqb, ret.Studio) if err != nil { return err } @@ -335,7 +334,7 @@ func (c Cache) postScrapeGallery(ret *models.ScrapedGallery) error { sqb := r.Studio() for _, p := range ret.Performers { - err := MatchScrapedScenePerformer(pqb, p) + err := MatchScrapedPerformer(pqb, p) if err != nil { return err } @@ -348,7 +347,7 @@ func (c Cache) postScrapeGallery(ret *models.ScrapedGallery) error { ret.Tags = tags if ret.Studio != nil { - err := MatchScrapedSceneStudio(sqb, ret.Studio) + err := MatchScrapedStudio(sqb, ret.Studio) if err != nil { return err } @@ -362,12 +361,55 @@ func (c Cache) postScrapeGallery(ret *models.ScrapedGallery) error { return nil } -// ScrapeScene uses the scraper with the provided ID to scrape a scene. -func (c Cache) ScrapeScene(scraperID string, scene models.SceneUpdateInput) (*models.ScrapedScene, error) { +// ScrapeScene uses the scraper with the provided ID to scrape a scene using existing data. +func (c Cache) ScrapeScene(scraperID string, sceneID int) (*models.ScrapedScene, error) { // find scraper with the provided id s := c.findScraper(scraperID) if s != nil { - ret, err := s.ScrapeScene(scene, c.txnManager, c.globalConfig) + // get scene from id + scene, err := getScene(sceneID, c.txnManager) + if err != nil { + return nil, err + } + + ret, err := s.ScrapeSceneByScene(scene, c.txnManager, c.globalConfig) + + if err != nil { + return nil, err + } + + if ret != nil { + err = c.postScrapeScene(ret) + if err != nil { + return nil, err + } + } + + return ret, nil + } + + return nil, errors.New("Scraper with ID " + scraperID + " not found") +} + +// ScrapeSceneQuery uses the scraper with the provided ID to query for +// scenes using the provided query string. It returns a list of +// scraped scene data. +func (c Cache) ScrapeSceneQuery(scraperID string, query string) ([]*models.ScrapedScene, error) { + // find scraper with the provided id + s := c.findScraper(scraperID) + if s != nil { + return s.ScrapeSceneQuery(query, c.txnManager, c.globalConfig) + } + + return nil, errors.New("Scraper with ID " + scraperID + " not found") +} + +// ScrapeSceneFragment uses the scraper with the provided ID to scrape a scene. +func (c Cache) ScrapeSceneFragment(scraperID string, scene models.ScrapedSceneInput) (*models.ScrapedScene, error) { + // find scraper with the provided id + s := c.findScraper(scraperID) + if s != nil { + ret, err := s.ScrapeSceneByFragment(scene, c.txnManager, c.globalConfig) if err != nil { return nil, err @@ -410,11 +452,40 @@ func (c Cache) ScrapeSceneURL(url string) (*models.ScrapedScene, error) { return nil, nil } -// ScrapeGallery uses the scraper with the provided ID to scrape a scene. -func (c Cache) ScrapeGallery(scraperID string, gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) { +// ScrapeGallery uses the scraper with the provided ID to scrape a gallery using existing data. +func (c Cache) ScrapeGallery(scraperID string, galleryID int) (*models.ScrapedGallery, error) { s := c.findScraper(scraperID) if s != nil { - ret, err := s.ScrapeGallery(gallery, c.txnManager, c.globalConfig) + // get gallery from id + gallery, err := getGallery(galleryID, c.txnManager) + if err != nil { + return nil, err + } + + ret, err := s.ScrapeGalleryByGallery(gallery, c.txnManager, c.globalConfig) + + if err != nil { + return nil, err + } + + if ret != nil { + err = c.postScrapeGallery(ret) + if err != nil { + return nil, err + } + } + + return ret, nil + } + + return nil, errors.New("Scraped with ID " + scraperID + " not found") +} + +// ScrapeGalleryFragment uses the scraper with the provided ID to scrape a gallery. +func (c Cache) ScrapeGalleryFragment(scraperID string, gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) { + s := c.findScraper(scraperID) + if s != nil { + ret, err := s.ScrapeGalleryByFragment(gallery, c.txnManager, c.globalConfig) if err != nil { return nil, err @@ -457,23 +528,6 @@ func (c Cache) ScrapeGalleryURL(url string) (*models.ScrapedGallery, error) { return nil, nil } -func matchMovieStudio(qb models.StudioReader, s *models.ScrapedMovieStudio) error { - studio, err := qb.FindByName(s.Name, true) - - if err != nil { - return err - } - - if studio == nil { - // ignore - cannot match - return nil - } - - id := strconv.Itoa(studio.ID) - s.ID = &id - return nil -} - // ScrapeMovieURL uses the first scraper it finds that matches the URL // provided to scrape a movie. If no scrapers are found that matches // the URL, then nil is returned. @@ -487,7 +541,7 @@ func (c Cache) ScrapeMovieURL(url string) (*models.ScrapedMovie, error) { if ret.Studio != nil { if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { - return matchMovieStudio(r.Studio(), ret.Studio) + return MatchScrapedStudio(r.Studio(), ret.Studio) }); err != nil { return nil, err } @@ -508,8 +562,8 @@ func (c Cache) ScrapeMovieURL(url string) (*models.ScrapedMovie, error) { return nil, nil } -func postProcessTags(tqb models.TagReader, scrapedTags []*models.ScrapedSceneTag) ([]*models.ScrapedSceneTag, error) { - var ret []*models.ScrapedSceneTag +func postProcessTags(tqb models.TagReader, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) { + var ret []*models.ScrapedTag excludePatterns := stash_config.GetInstance().GetScraperExcludeTagPatterns() var excludeRegexps []*regexp.Regexp @@ -533,7 +587,7 @@ ScrapeTag: } } - err := MatchScrapedSceneTag(tqb, t) + err := MatchScrapedTag(tqb, t) if err != nil { return nil, err } diff --git a/pkg/scraper/script.go b/pkg/scraper/script.go index 32f768d45..a53266e0e 100644 --- a/pkg/scraper/script.go +++ b/pkg/scraper/script.go @@ -1,9 +1,9 @@ package scraper import ( - "bufio" "encoding/json" "errors" + "fmt" "io" "os/exec" "path/filepath" @@ -48,7 +48,9 @@ func (s *scriptScraper) runScraperScript(inString string, out interface{}) error go func() { defer stdin.Close() - io.WriteString(stdin, inString) + if n, err := io.WriteString(stdin, inString); err != nil { + logger.Warnf("failure to write full input to script (wrote %v bytes out of %v): %v", n, len(inString), err) + } }() stderr, err := cmd.StderrPipe() @@ -63,15 +65,10 @@ func (s *scriptScraper) runScraperScript(inString string, out interface{}) error if err = cmd.Start(); err != nil { logger.Error("Error running scraper script: " + err.Error()) - return errors.New("Error running scraper script") + return errors.New("error running scraper script") } - scanner := bufio.NewScanner(stderr) - go func() { // log errors from stderr pipe - for scanner.Scan() { - logger.Errorf("scraper: %s", scanner.Text()) - } - }() + go handleScraperStderr(s.config.Name, stderr) logger.Debugf("Scraper script <%s> started", strings.Join(cmd.Args, " ")) @@ -86,7 +83,7 @@ func (s *scriptScraper) runScraperScript(inString string, out interface{}) error logger.Debugf("Scraper script finished") if err != nil { - return errors.New("Error running scraper script") + return errors.New("error running scraper script") } return nil @@ -134,7 +131,39 @@ func (s *scriptScraper) scrapePerformerByURL(url string) (*models.ScrapedPerform return &ret, err } -func (s *scriptScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) { +func (s *scriptScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) { + inString, err := json.Marshal(sceneToUpdateInput(scene)) + + if err != nil { + return nil, err + } + + var ret models.ScrapedScene + + err = s.runScraperScript(string(inString), &ret) + + return &ret, err +} + +func (s *scriptScraper) scrapeScenesByName(name string) ([]*models.ScrapedScene, error) { + inString := `{"name": "` + name + `"}` + + var scenes []models.ScrapedScene + + err := s.runScraperScript(inString, &scenes) + + // convert to pointers + var ret []*models.ScrapedScene + if err == nil { + for i := 0; i < len(scenes); i++ { + ret = append(ret, &scenes[i]) + } + } + + return ret, err +} + +func (s *scriptScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) { inString, err := json.Marshal(scene) if err != nil { @@ -148,7 +177,21 @@ func (s *scriptScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*m return &ret, err } -func (s *scriptScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) { +func (s *scriptScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) { + inString, err := json.Marshal(galleryToUpdateInput(gallery)) + + if err != nil { + return nil, err + } + + var ret models.ScrapedGallery + + err = s.runScraperScript(string(inString), &ret) + + return &ret, err +} + +func (s *scriptScraper) scrapeGalleryByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) { inString, err := json.Marshal(gallery) if err != nil { @@ -207,3 +250,13 @@ func findPythonExecutable() (string, error) { return "python3", nil } + +func handleScraperStderr(name string, scraperOutputReader io.ReadCloser) { + const scraperPrefix = "[Scrape / %s] " + + lgr := logger.PluginLogger{ + Prefix: fmt.Sprintf(scraperPrefix, name), + DefaultLogLevel: &logger.ErrorLevel, + } + lgr.HandlePluginStdErr(scraperOutputReader) +} diff --git a/pkg/scraper/stash.go b/pkg/scraper/stash.go index d37b82847..7023a6d3a 100644 --- a/pkg/scraper/stash.go +++ b/pkg/scraper/stash.go @@ -2,6 +2,7 @@ package scraper import ( "context" + "database/sql" "errors" "strconv" @@ -81,11 +82,40 @@ func (s *stashScraper) scrapePerformersByName(name string) ([]*models.ScrapedPer return ret, nil } +// need a separate for scraped stash performers - does not include remote_site_id or image +type scrapedTagStash struct { + Name string `graphql:"name" json:"name"` +} + +type scrapedPerformerStash struct { + Name *string `graphql:"name" json:"name"` + Gender *string `graphql:"gender" json:"gender"` + URL *string `graphql:"url" json:"url"` + Twitter *string `graphql:"twitter" json:"twitter"` + Instagram *string `graphql:"instagram" json:"instagram"` + Birthdate *string `graphql:"birthdate" json:"birthdate"` + Ethnicity *string `graphql:"ethnicity" json:"ethnicity"` + Country *string `graphql:"country" json:"country"` + EyeColor *string `graphql:"eye_color" json:"eye_color"` + Height *string `graphql:"height" json:"height"` + Measurements *string `graphql:"measurements" json:"measurements"` + FakeTits *string `graphql:"fake_tits" json:"fake_tits"` + CareerLength *string `graphql:"career_length" json:"career_length"` + Tattoos *string `graphql:"tattoos" json:"tattoos"` + Piercings *string `graphql:"piercings" json:"piercings"` + Aliases *string `graphql:"aliases" json:"aliases"` + Tags []*scrapedTagStash `graphql:"tags" json:"tags"` + Details *string `graphql:"details" json:"details"` + DeathDate *string `graphql:"death_date" json:"death_date"` + HairColor *string `graphql:"hair_color" json:"hair_color"` + Weight *string `graphql:"weight" json:"weight"` +} + func (s *stashScraper) scrapePerformerByFragment(scrapedPerformer models.ScrapedPerformerInput) (*models.ScrapedPerformer, error) { client := s.getStashClient() var q struct { - FindPerformer *models.ScrapedPerformerStash `graphql:"findPerformer(id: $f)"` + FindPerformer *scrapedPerformerStash `graphql:"findPerformer(id: $f)"` } performerID := *scrapedPerformer.URL @@ -100,13 +130,6 @@ func (s *stashScraper) scrapePerformerByFragment(scrapedPerformer models.Scraped return nil, err } - if q.FindPerformer != nil { - // the ids of the tags must be nilled - for _, t := range q.FindPerformer.Tags { - t.ID = nil - } - } - // need to copy back to a scraped performer ret := models.ScrapedPerformer{} err = copier.Copy(&ret, q.FindPerformer) @@ -123,25 +146,83 @@ func (s *stashScraper) scrapePerformerByFragment(scrapedPerformer models.Scraped return &ret, nil } -func (s *stashScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) { - // query by MD5 - // assumes that the scene exists in the database - id, err := strconv.Atoi(scene.ID) +type scrapedStudioStash struct { + Name string `graphql:"name" json:"name"` + URL *string `graphql:"url" json:"url"` +} + +type stashFindSceneNamesResultType struct { + Count int `graphql:"count"` + Scenes []*scrapedSceneStash `graphql:"scenes"` +} + +func (s *stashScraper) scrapedStashSceneToScrapedScene(scene *scrapedSceneStash) (*models.ScrapedScene, error) { + ret := models.ScrapedScene{} + err := copier.Copy(&ret, scene) if err != nil { return nil, err } - var storedScene *models.Scene - if err := s.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { - var err error - storedScene, err = r.Scene().Find(id) - return err - }); err != nil { + // get the performer image directly + ret.Image, err = getStashSceneImage(s.config.StashServer.URL, scene.ID, s.globalConfig) + if err != nil { return nil, err } + return &ret, nil +} + +func (s *stashScraper) scrapeScenesByName(name string) ([]*models.ScrapedScene, error) { + client := s.getStashClient() + var q struct { - FindScene *models.ScrapedSceneStash `graphql:"findSceneByHash(input: $c)"` + FindScenes stashFindSceneNamesResultType `graphql:"findScenes(filter: $f)"` + } + + page := 1 + perPage := 10 + + vars := map[string]interface{}{ + "f": models.FindFilterType{ + Q: &name, + Page: &page, + PerPage: &perPage, + }, + } + + err := client.Query(context.Background(), &q, vars) + if err != nil { + return nil, err + } + + var ret []*models.ScrapedScene + for _, scene := range q.FindScenes.Scenes { + converted, err := s.scrapedStashSceneToScrapedScene(scene) + if err != nil { + return nil, err + } + ret = append(ret, converted) + } + + return ret, nil +} + +type scrapedSceneStash struct { + ID string `graphql:"id" json:"id"` + Title *string `graphql:"title" json:"title"` + Details *string `graphql:"details" json:"details"` + URL *string `graphql:"url" json:"url"` + Date *string `graphql:"date" json:"date"` + File *models.SceneFileType `graphql:"file" json:"file"` + Studio *scrapedStudioStash `graphql:"studio" json:"studio"` + Tags []*scrapedTagStash `graphql:"tags" json:"tags"` + Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"` +} + +func (s *stashScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) { + // query by MD5 + var q struct { + FindScene *scrapedSceneStash `graphql:"findSceneByHash(input: $c)"` } type SceneHashInput struct { @@ -150,8 +231,8 @@ func (s *stashScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mo } input := SceneHashInput{ - Checksum: &storedScene.Checksum.String, - Oshash: &storedScene.OSHash.String, + Checksum: &scene.Checksum.String, + Oshash: &scene.OSHash.String, } vars := map[string]interface{}{ @@ -159,29 +240,12 @@ func (s *stashScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mo } client := s.getStashClient() - err = client.Query(context.Background(), &q, vars) - if err != nil { + if err := client.Query(context.Background(), &q, vars); err != nil { return nil, err } - if q.FindScene != nil { - // the ids of the studio, performers and tags must be nilled - if q.FindScene.Studio != nil { - q.FindScene.Studio.ID = nil - } - - for _, p := range q.FindScene.Performers { - p.ID = nil - } - - for _, t := range q.FindScene.Tags { - t.ID = nil - } - } - // need to copy back to a scraped scene - ret := models.ScrapedScene{} - err = copier.Copy(&ret, q.FindScene) + ret, err := s.scrapedStashSceneToScrapedScene(q.FindScene) if err != nil { return nil, err } @@ -192,30 +256,28 @@ func (s *stashScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mo return nil, err } - return &ret, nil + return ret, nil } -func (s *stashScraper) scrapeGalleryByFragment(scene models.GalleryUpdateInput) (*models.ScrapedGallery, error) { - id, err := strconv.Atoi(scene.ID) - if err != nil { - return nil, err - } +func (s *stashScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) { + return nil, errors.New("scrapeSceneByFragment not supported for stash scraper") +} - // query by MD5 - // assumes that the gallery exists in the database - var storedGallery *models.Gallery - if err := s.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { - qb := r.Gallery() - - var err error - storedGallery, err = qb.Find(id) - return err - }); err != nil { - return nil, err - } +type scrapedGalleryStash struct { + ID string `graphql:"id" json:"id"` + Title *string `graphql:"title" json:"title"` + Details *string `graphql:"details" json:"details"` + URL *string `graphql:"url" json:"url"` + Date *string `graphql:"date" json:"date"` + File *models.SceneFileType `graphql:"file" json:"file"` + Studio *scrapedStudioStash `graphql:"studio" json:"studio"` + Tags []*scrapedTagStash `graphql:"tags" json:"tags"` + Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"` +} +func (s *stashScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) { var q struct { - FindGallery *models.ScrapedGalleryStash `graphql:"findGalleryByHash(input: $c)"` + FindGallery *scrapedGalleryStash `graphql:"findGalleryByHash(input: $c)"` } type GalleryHashInput struct { @@ -223,7 +285,7 @@ func (s *stashScraper) scrapeGalleryByFragment(scene models.GalleryUpdateInput) } input := GalleryHashInput{ - Checksum: &storedGallery.Checksum, + Checksum: &gallery.Checksum, } vars := map[string]interface{}{ @@ -231,36 +293,23 @@ func (s *stashScraper) scrapeGalleryByFragment(scene models.GalleryUpdateInput) } client := s.getStashClient() - err = client.Query(context.Background(), &q, vars) - if err != nil { + if err := client.Query(context.Background(), &q, vars); err != nil { return nil, err } - if q.FindGallery != nil { - // the ids of the studio, performers and tags must be nilled - if q.FindGallery.Studio != nil { - q.FindGallery.Studio.ID = nil - } - - for _, p := range q.FindGallery.Performers { - p.ID = nil - } - - for _, t := range q.FindGallery.Tags { - t.ID = nil - } - } - // need to copy back to a scraped scene ret := models.ScrapedGallery{} - err = copier.Copy(&ret, q.FindGallery) - if err != nil { + if err := copier.Copy(&ret, q.FindGallery); err != nil { return nil, err } return &ret, nil } +func (s *stashScraper) scrapeGalleryByFragment(scene models.ScrapedGalleryInput) (*models.ScrapedGallery, error) { + return nil, errors.New("scrapeGalleryByFragment not supported for stash scraper") +} + func (s *stashScraper) scrapePerformerByURL(url string) (*models.ScrapedPerformer, error) { return nil, errors.New("scrapePerformerByURL not supported for stash scraper") } @@ -277,17 +326,11 @@ func (s *stashScraper) scrapeMovieByURL(url string) (*models.ScrapedMovie, error return nil, errors.New("scrapeMovieByURL not supported for stash scraper") } -func sceneFromUpdateFragment(scene models.SceneUpdateInput, txnManager models.TransactionManager) (*models.Scene, error) { - id, err := strconv.Atoi(scene.ID) - if err != nil { - return nil, err - } - - // TODO - should we modify it with the input? +func getScene(sceneID int, txnManager models.TransactionManager) (*models.Scene, error) { var ret *models.Scene if err := txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { var err error - ret, err = r.Scene().Find(id) + ret, err = r.Scene().Find(sceneID) return err }); err != nil { return nil, err @@ -295,18 +338,66 @@ func sceneFromUpdateFragment(scene models.SceneUpdateInput, txnManager models.Tr return ret, nil } -func galleryFromUpdateFragment(gallery models.GalleryUpdateInput, txnManager models.TransactionManager) (ret *models.Gallery, err error) { - id, err := strconv.Atoi(gallery.ID) - if err != nil { - return nil, err +func sceneToUpdateInput(scene *models.Scene) models.SceneUpdateInput { + toStringPtr := func(s sql.NullString) *string { + if s.Valid { + return &s.String + } + + return nil } + dateToStringPtr := func(s models.SQLiteDate) *string { + if s.Valid { + return &s.String + } + + return nil + } + + return models.SceneUpdateInput{ + ID: strconv.Itoa(scene.ID), + Title: toStringPtr(scene.Title), + Details: toStringPtr(scene.Details), + URL: toStringPtr(scene.URL), + Date: dateToStringPtr(scene.Date), + } +} + +func getGallery(galleryID int, txnManager models.TransactionManager) (*models.Gallery, error) { + var ret *models.Gallery if err := txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { - ret, err = r.Gallery().Find(id) + var err error + ret, err = r.Gallery().Find(galleryID) return err }); err != nil { return nil, err } - return ret, nil } + +func galleryToUpdateInput(gallery *models.Gallery) models.GalleryUpdateInput { + toStringPtr := func(s sql.NullString) *string { + if s.Valid { + return &s.String + } + + return nil + } + + dateToStringPtr := func(s models.SQLiteDate) *string { + if s.Valid { + return &s.String + } + + return nil + } + + return models.GalleryUpdateInput{ + ID: strconv.Itoa(gallery.ID), + Title: toStringPtr(gallery.Title), + Details: toStringPtr(gallery.Details), + URL: toStringPtr(gallery.URL), + Date: dateToStringPtr(gallery.Date), + } +} diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 4ab848fbf..b6174d7ee 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -3,7 +3,7 @@ package stashbox import ( "context" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "strings" @@ -66,8 +66,79 @@ func (c Client) QueryStashBoxScene(queryStr string) ([]*models.ScrapedScene, err } // FindStashBoxScenesByFingerprints queries stash-box for scenes using every -// scene's MD5/OSHASH checksum, or PHash -func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([]*models.ScrapedScene, error) { +// scene's MD5/OSHASH checksum, or PHash, and returns results in the same order +// as the input slice. +func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([][]*models.ScrapedScene, error) { + ids, err := utils.StringSliceToIntSlice(sceneIDs) + if err != nil { + return nil, err + } + + var fingerprints []string + // map fingerprints to their scene index + fpToScene := make(map[string][]int) + + if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + qb := r.Scene() + + for index, sceneID := range ids { + scene, err := qb.Find(sceneID) + if err != nil { + return err + } + + if scene == nil { + return fmt.Errorf("scene with id %d not found", sceneID) + } + + if scene.Checksum.Valid { + fingerprints = append(fingerprints, scene.Checksum.String) + fpToScene[scene.Checksum.String] = append(fpToScene[scene.Checksum.String], index) + } + + if scene.OSHash.Valid { + fingerprints = append(fingerprints, scene.OSHash.String) + fpToScene[scene.OSHash.String] = append(fpToScene[scene.OSHash.String], index) + } + + if scene.Phash.Valid { + phashStr := utils.PhashToString(scene.Phash.Int64) + fingerprints = append(fingerprints, phashStr) + fpToScene[phashStr] = append(fpToScene[phashStr], index) + } + } + + return nil + }); err != nil { + return nil, err + } + + allScenes, err := c.findStashBoxScenesByFingerprints(fingerprints) + if err != nil { + return nil, err + } + + // set the matched scenes back in their original order + ret := make([][]*models.ScrapedScene, len(sceneIDs)) + for _, s := range allScenes { + var addedTo []int + for _, fp := range s.Fingerprints { + sceneIndexes := fpToScene[fp.Hash] + for _, index := range sceneIndexes { + if !utils.IntInclude(addedTo, index) { + addedTo = append(addedTo, index) + ret[index] = append(ret[index], s) + } + } + } + } + + return ret, nil +} + +// FindStashBoxScenesByFingerprintsFlat queries stash-box for scenes using every +// scene's MD5/OSHASH checksum, or PHash, and returns results a flat slice. +func (c Client) FindStashBoxScenesByFingerprintsFlat(sceneIDs []string) ([]*models.ScrapedScene, error) { ids, err := utils.StringSliceToIntSlice(sceneIDs) if err != nil { return nil, err @@ -97,7 +168,8 @@ func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([]*models.S } if scene.Phash.Valid { - fingerprints = append(fingerprints, utils.PhashToString(scene.Phash.Int64)) + phashStr := utils.PhashToString(scene.Phash.Int64) + fingerprints = append(fingerprints, phashStr) } } @@ -237,10 +309,18 @@ func (c Client) QueryStashBoxPerformer(queryStr string) ([]*models.StashBoxPerfo Results: performers, }, } + + // set the deprecated image field + for _, p := range res[0].Results { + if len(p.Images) > 0 { + p.Image = &p.Images[0] + } + } + return res, err } -func (c Client) queryStashBoxPerformer(queryStr string) ([]*models.ScrapedScenePerformer, error) { +func (c Client) queryStashBoxPerformer(queryStr string) ([]*models.ScrapedPerformer, error) { performers, err := c.client.SearchPerformer(context.TODO(), queryStr) if err != nil { return nil, err @@ -248,7 +328,7 @@ func (c Client) queryStashBoxPerformer(queryStr string) ([]*models.ScrapedSceneP performerFragments := performers.SearchPerformer - var ret []*models.ScrapedScenePerformer + var ret []*models.ScrapedPerformer for _, fragment := range performerFragments { performer := performerFragmentToScrapedScenePerformer(*fragment) ret = append(ret, performer) @@ -292,6 +372,50 @@ func (c Client) FindStashBoxPerformersByNames(performerIDs []string) ([]*models. return c.findStashBoxPerformersByNames(performers) } +func (c Client) FindStashBoxPerformersByPerformerNames(performerIDs []string) ([][]*models.ScrapedPerformer, error) { + ids, err := utils.StringSliceToIntSlice(performerIDs) + if err != nil { + return nil, err + } + + var performers []*models.Performer + + if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + qb := r.Performer() + + for _, performerID := range ids { + performer, err := qb.Find(performerID) + if err != nil { + return err + } + + if performer == nil { + return fmt.Errorf("performer with id %d not found", performerID) + } + + if performer.Name.Valid { + performers = append(performers, performer) + } + } + + return nil + }); err != nil { + return nil, err + } + + results, err := c.findStashBoxPerformersByNames(performers) + if err != nil { + return nil, err + } + + var ret [][]*models.ScrapedPerformer + for _, r := range results { + ret = append(ret, r.Results) + } + + return ret, nil +} + func (c Client) findStashBoxPerformersByNames(performers []*models.Performer) ([]*models.StashBoxPerformerQueryResult, error) { var ret []*models.StashBoxPerformerQueryResult for _, performer := range performers { @@ -398,7 +522,7 @@ func fetchImage(url string) (*string, error) { defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } @@ -413,14 +537,14 @@ func fetchImage(url string) (*string, error) { return &img, nil } -func performerFragmentToScrapedScenePerformer(p graphql.PerformerFragment) *models.ScrapedScenePerformer { +func performerFragmentToScrapedScenePerformer(p graphql.PerformerFragment) *models.ScrapedPerformer { id := p.ID images := []string{} for _, image := range p.Images { images = append(images, image.URL) } - sp := &models.ScrapedScenePerformer{ - Name: p.Name, + sp := &models.ScrapedPerformer{ + Name: &p.Name, Country: p.Country, Measurements: formatMeasurements(p.Measurements), CareerLength: formatCareerLength(p.CareerStartYear, p.CareerEndYear), @@ -430,10 +554,13 @@ func performerFragmentToScrapedScenePerformer(p graphql.PerformerFragment) *mode RemoteSiteID: &id, Images: images, // TODO - tags not currently supported - // TODO - Image - should be returned as a set of URLs. Will need a // graphql schema change to accommodate this. Leave off for now. } + if len(sp.Images) > 0 { + sp.Image = &sp.Images[0] + } + if p.Height != nil && *p.Height > 0 { hs := strconv.Itoa(*p.Height) sp.Height = &hs @@ -511,13 +638,13 @@ func sceneFragmentToScrapedScene(txnManager models.TransactionManager, s *graphq if s.Studio != nil { studioID := s.Studio.ID - ss.Studio = &models.ScrapedSceneStudio{ + ss.Studio = &models.ScrapedStudio{ Name: s.Studio.Name, URL: findURL(s.Studio.Urls, "HOME"), RemoteSiteID: &studioID, } - err := scraper.MatchScrapedSceneStudio(r.Studio(), ss.Studio) + err := scraper.MatchScrapedStudio(r.Studio(), ss.Studio) if err != nil { return err } @@ -526,7 +653,7 @@ func sceneFragmentToScrapedScene(txnManager models.TransactionManager, s *graphq for _, p := range s.Performers { sp := performerFragmentToScrapedScenePerformer(p.Performer) - err := scraper.MatchScrapedScenePerformer(pqb, sp) + err := scraper.MatchScrapedPerformer(pqb, sp) if err != nil { return err } @@ -535,11 +662,11 @@ func sceneFragmentToScrapedScene(txnManager models.TransactionManager, s *graphq } for _, t := range s.Tags { - st := &models.ScrapedSceneTag{ + st := &models.ScrapedTag{ Name: t.Name, } - err := scraper.MatchScrapedSceneTag(tqb, st) + err := scraper.MatchScrapedTag(tqb, st) if err != nil { return err } @@ -555,7 +682,7 @@ func sceneFragmentToScrapedScene(txnManager models.TransactionManager, s *graphq return ss, nil } -func (c Client) FindStashBoxPerformerByID(id string) (*models.ScrapedScenePerformer, error) { +func (c Client) FindStashBoxPerformerByID(id string) (*models.ScrapedPerformer, error) { performer, err := c.client.FindPerformerByID(context.TODO(), id) if err != nil { return nil, err @@ -565,13 +692,13 @@ func (c Client) FindStashBoxPerformerByID(id string) (*models.ScrapedScenePerfor return ret, nil } -func (c Client) FindStashBoxPerformerByName(name string) (*models.ScrapedScenePerformer, error) { +func (c Client) FindStashBoxPerformerByName(name string) (*models.ScrapedPerformer, error) { performers, err := c.client.SearchPerformer(context.TODO(), name) if err != nil { return nil, err } - var ret *models.ScrapedScenePerformer + var ret *models.ScrapedPerformer for _, performer := range performers.SearchPerformer { if strings.EqualFold(performer.Name, name) { ret = performerFragmentToScrapedScenePerformer(*performer) diff --git a/pkg/scraper/url.go b/pkg/scraper/url.go index 4fc78caf9..fe33c157d 100644 --- a/pkg/scraper/url.go +++ b/pkg/scraper/url.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "net/http/cookiejar" "os" @@ -92,7 +91,7 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } @@ -109,7 +108,7 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re func urlFromCDP(url string, driverOptions scraperDriverOptions, globalConfig GlobalConfig) (io.Reader, error) { if !driverOptions.UseCDP { - return nil, fmt.Errorf("Url shouldn't be feetched through CDP") + return nil, fmt.Errorf("url shouldn't be fetched through CDP") } sleepDuration := scrapeDefaultSleep @@ -140,7 +139,7 @@ func urlFromCDP(url string, driverOptions scraperDriverOptions, globalConfig Glo act, cancelAct = chromedp.NewRemoteAllocator(context.Background(), remote) } else { // use a temporary user directory for chrome - dir, err := ioutil.TempDir("", "stash-chromedp") + dir, err := os.MkdirTemp("", "stash-chromedp") if err != nil { return nil, err } @@ -224,6 +223,7 @@ func getRemoteCDPWSAddress(address string) (string, error) { if err != nil { return "", err } + defer resp.Body.Close() var result map[string]interface{} var json = jsoniter.ConfigCompatibleWithStandardLibrary @@ -235,17 +235,6 @@ func getRemoteCDPWSAddress(address string) (string, error) { return remote, err } -func cdpNetwork(enable bool) chromedp.Action { - return chromedp.ActionFunc(func(ctx context.Context) error { - if enable { - network.Enable().Do(ctx) - } else { - network.Disable().Do(ctx) - } - return nil - }) -} - func cdpHeaders(driverOptions scraperDriverOptions) map[string]interface{} { headers := map[string]interface{}{} if driverOptions.Headers != nil { diff --git a/pkg/scraper/xpath.go b/pkg/scraper/xpath.go index e612b5f4d..0cd955788 100644 --- a/pkg/scraper/xpath.go +++ b/pkg/scraper/xpath.go @@ -124,18 +124,34 @@ func (s *xpathScraper) scrapePerformerByFragment(scrapedPerformer models.Scraped return nil, errors.New("scrapePerformerByFragment not supported for xpath scraper") } -func (s *xpathScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) { - storedScene, err := sceneFromUpdateFragment(scene, s.txnManager) +func (s *xpathScraper) scrapeScenesByName(name string) ([]*models.ScrapedScene, error) { + scraper := s.getXpathScraper() + + if scraper == nil { + return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config") + } + + const placeholder = "{}" + + // replace the placeholder string with the URL-escaped name + escapedName := url.QueryEscape(name) + + url := s.scraper.QueryURL + url = strings.Replace(url, placeholder, escapedName, -1) + + doc, err := s.loadURL(url) + if err != nil { return nil, err } - if storedScene == nil { - return nil, errors.New("no scene found") - } + q := s.getXPathQuery(doc) + return scraper.scrapeScenes(q) +} +func (s *xpathScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) { // construct the URL - queryURL := queryURLParametersFromScene(storedScene) + queryURL := queryURLParametersFromScene(scene) if s.scraper.QueryURLReplacements != nil { queryURL.applyReplacements(s.scraper.QueryURLReplacements) } @@ -157,18 +173,33 @@ func (s *xpathScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mo return scraper.scrapeScene(q) } -func (s *xpathScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) { - storedGallery, err := galleryFromUpdateFragment(gallery, s.txnManager) +func (s *xpathScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) { + // construct the URL + queryURL := queryURLParametersFromScrapedScene(scene) + if s.scraper.QueryURLReplacements != nil { + queryURL.applyReplacements(s.scraper.QueryURLReplacements) + } + url := queryURL.constructURL(s.scraper.QueryURL) + + scraper := s.getXpathScraper() + + if scraper == nil { + return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config") + } + + doc, err := s.loadURL(url) + if err != nil { return nil, err } - if storedGallery == nil { - return nil, errors.New("no scene found") - } + q := s.getXPathQuery(doc) + return scraper.scrapeScene(q) +} +func (s *xpathScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) { // construct the URL - queryURL := queryURLParametersFromGallery(storedGallery) + queryURL := queryURLParametersFromGallery(gallery) if s.scraper.QueryURLReplacements != nil { queryURL.applyReplacements(s.scraper.QueryURLReplacements) } @@ -190,6 +221,10 @@ func (s *xpathScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput return scraper.scrapeGallery(q) } +func (s *xpathScraper) scrapeGalleryByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) { + return nil, errors.New("scrapeGalleryByFragment not supported for xpath scraper") +} + func (s *xpathScraper) loadURL(url string) (*html.Node, error) { r, err := loadURL(url, s.config, s.globalConfig) if err != nil { @@ -200,7 +235,9 @@ func (s *xpathScraper) loadURL(url string) (*html.Node, error) { if err == nil && s.config.DebugOptions != nil && s.config.DebugOptions.PrintHTML { var b bytes.Buffer - html.Render(&b, ret) + if err := html.Render(&b, ret); err != nil { + logger.Warnf("could not render HTML: %v", err) + } logger.Infof("loadURL (%s) response: \n%s", url, b.String()) } diff --git a/pkg/scraper/xpath_test.go b/pkg/scraper/xpath_test.go index 5983bd7a0..4ba98d40d 100644 --- a/pkg/scraper/xpath_test.go +++ b/pkg/scraper/xpath_test.go @@ -53,9 +53,9 @@ const htmlDoc1 = ` Country of Origin: - + - + United States @@ -593,7 +593,7 @@ func makeSceneXPathConfig() mappedScraper { return scraper } -func verifyTags(t *testing.T, expectedTagNames []string, actualTags []*models.ScrapedSceneTag) { +func verifyTags(t *testing.T, expectedTagNames []string, actualTags []*models.ScrapedTag) { t.Helper() i := 0 @@ -614,7 +614,7 @@ func verifyTags(t *testing.T, expectedTagNames []string, actualTags []*models.Sc } } -func verifyMovies(t *testing.T, expectedMovieNames []string, actualMovies []*models.ScrapedSceneMovie) { +func verifyMovies(t *testing.T, expectedMovieNames []string, actualMovies []*models.ScrapedMovie) { t.Helper() i := 0 @@ -625,7 +625,7 @@ func verifyMovies(t *testing.T, expectedMovieNames []string, actualMovies []*mod expectedMovie = expectedMovieNames[i] } if i < len(actualMovies) { - actualMovie = actualMovies[i].Name + actualMovie = *actualMovies[i].Name } if expectedMovie != actualMovie { @@ -635,7 +635,7 @@ func verifyMovies(t *testing.T, expectedMovieNames []string, actualMovies []*mod } } -func verifyPerformers(t *testing.T, expectedNames []string, expectedURLs []string, actualPerformers []*models.ScrapedScenePerformer) { +func verifyPerformers(t *testing.T, expectedNames []string, expectedURLs []string, actualPerformers []*models.ScrapedPerformer) { t.Helper() i := 0 @@ -651,7 +651,7 @@ func verifyPerformers(t *testing.T, expectedNames []string, expectedURLs []strin expectedURL = expectedURLs[i] } if i < len(actualPerformers) { - actualName = actualPerformers[i].Name + actualName = *actualPerformers[i].Name if actualPerformers[i].URL != nil { actualURL = *actualPerformers[i].URL } @@ -661,7 +661,7 @@ func verifyPerformers(t *testing.T, expectedNames []string, expectedURLs []strin t.Errorf("Expected performer name %s, got %s", expectedName, actualName) } if expectedURL != actualURL { - t.Errorf("Expected perfromer URL %s, got %s", expectedName, actualName) + t.Errorf("Expected performer URL %s, got %s", expectedName, actualName) } i++ } @@ -741,7 +741,7 @@ func TestLoadXPathScraperFromYAML(t *testing.T) { const yamlStr = `name: Test performerByURL: - action: scrapeXPath - url: + url: - test.com scraper: performerScraper xPathScrapers: @@ -755,11 +755,11 @@ xPathScrapers: postProcess: - parseDate: January 2, 2006 Tags: - Name: //tags + Name: //tags Movies: - Name: //movies + Name: //movies Performers: - Name: //performers + Name: //performers Studio: Name: //studio ` @@ -848,13 +848,13 @@ func TestSubScrape(t *testing.T) { yamlStr := `name: Test performerByURL: - action: scrapeXPath - url: + url: - ` + ts.URL + ` scraper: performerScraper xPathScrapers: performerScraper: performer: - Name: + Name: selector: //div/a/@href postProcess: - replace: diff --git a/pkg/session/authentication.go b/pkg/session/authentication.go new file mode 100644 index 000000000..7bdf0ea22 --- /dev/null +++ b/pkg/session/authentication.go @@ -0,0 +1,120 @@ +package session + +import ( + "fmt" + "net" + "net/http" + "strings" + + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/manager/config" +) + +type ExternalAccessError net.IP + +func (e ExternalAccessError) Error() string { + return fmt.Sprintf("stash accessed from external IP %s", net.IP(e).String()) +} + +type UntrustedProxyError net.IP + +func (e UntrustedProxyError) Error() string { + return fmt.Sprintf("untrusted proxy %s", net.IP(e).String()) +} + +func CheckAllowPublicWithoutAuth(c *config.Instance, r *http.Request) error { + if !c.HasCredentials() && !c.GetDangerousAllowPublicWithoutAuth() && !c.IsNewSystem() { + requestIPString, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return fmt.Errorf("error parsing remote host (%s): %w", r.RemoteAddr, err) + } + + requestIP := net.ParseIP(requestIPString) + + if r.Header.Get("X-FORWARDED-FOR") != "" { + // Request was proxied + trustedProxies := c.GetTrustedProxies() + proxyChain := strings.Split(r.Header.Get("X-FORWARDED-FOR"), ", ") + + if len(trustedProxies) == 0 { + // validate proxies against local network only + if !isLocalIP(requestIP) { + return ExternalAccessError(requestIP) + } else { + // Safe to validate X-Forwarded-For + for i := range proxyChain { + ip := net.ParseIP(proxyChain[i]) + if !isLocalIP(ip) { + return ExternalAccessError(ip) + } + } + } + } else { + // validate proxies against trusted proxies list + if isIPTrustedProxy(requestIP, trustedProxies) { + // Safe to validate X-Forwarded-For + // validate backwards, as only the last one is not attacker-controlled + for i := len(proxyChain) - 1; i >= 0; i-- { + ip := net.ParseIP(proxyChain[i]) + if i == 0 { + // last entry is originating device, check if from the public internet + if !isLocalIP(ip) { + return ExternalAccessError(ip) + } + } else if !isIPTrustedProxy(ip, trustedProxies) { + return UntrustedProxyError(ip) + } + } + } else { + // Proxy not on safe proxy list + return UntrustedProxyError(requestIP) + } + } + } else { + // request was not proxied + if !isLocalIP(requestIP) { + return ExternalAccessError(requestIP) + } + } + } + + return nil +} + +func CheckExternalAccessTripwire(c *config.Instance) *ExternalAccessError { + if !c.HasCredentials() && !c.GetDangerousAllowPublicWithoutAuth() { + if remoteIP := c.GetSecurityTripwireAccessedFromPublicInternet(); remoteIP != "" { + err := ExternalAccessError(net.ParseIP(remoteIP)) + return &err + } + } + + return nil +} + +func isLocalIP(requestIP net.IP) bool { + _, cgNatAddrSpace, _ := net.ParseCIDR("100.64.0.0/10") + return requestIP.IsPrivate() || requestIP.IsLoopback() || cgNatAddrSpace.Contains(requestIP) +} + +func isIPTrustedProxy(ip net.IP, trustedProxies []string) bool { + if len(trustedProxies) == 0 { + return isLocalIP(ip) + } + for _, v := range trustedProxies { + if ip.Equal(net.ParseIP(v)) { + return true + } + } + return false +} + +func LogExternalAccessError(err ExternalAccessError) { + logger.Errorf("Stash has been accessed from the internet (public IP %s), without authentication. \n"+ + "This is extremely dangerous! The whole world can see your stash page and browse your files! \n"+ + "You probably forwarded a port from your router. At the very least, add a password to stash in the settings. \n"+ + "Stash will not serve requests until you edit config.yml, remove the security_tripwire_accessed_from_public_internet key and restart stash. \n"+ + "This behaviour can be overridden (but not recommended) by setting dangerous_allow_public_without_auth to true in config.yml. \n"+ + "More information is available at https://github.com/stashapp/stash/wiki/Authentication-Required-When-Accessing-Stash-From-the-Internet \n"+ + "Stash is not answering any other requests to protect your privacy.", net.IP(err).String()) +} diff --git a/pkg/session/session.go b/pkg/session/session.go index 55faa4282..9d754e63a 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -173,19 +173,6 @@ func setVisitedPlugins(ctx context.Context, visitedPlugins []string) context.Con return context.WithValue(ctx, contextVisitedPlugins, visitedPlugins) } -func (s *Store) createSessionCookie(username string) (*http.Cookie, error) { - session := sessions.NewSession(s.sessionStore, cookieName) - session.Values[userIDKey] = username - - encoded, err := securecookie.EncodeMulti(session.Name(), session.Values, - s.sessionStore.Codecs...) - if err != nil { - return nil, err - } - - return sessions.NewCookie(session.Name(), encoded, session.Options), nil -} - func (s *Store) MakePluginCookie(ctx context.Context) *http.Cookie { currentUser := GetCurrentUserID(ctx) visitedPlugins := GetVisitedPlugins(ctx) diff --git a/pkg/sqlite/filter.go b/pkg/sqlite/filter.go index 1517bf99a..c85b68536 100644 --- a/pkg/sqlite/filter.go +++ b/pkg/sqlite/filter.go @@ -4,8 +4,11 @@ import ( "errors" "fmt" "regexp" + "strconv" "strings" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) @@ -97,11 +100,12 @@ type filterBuilder struct { whereClauses []sqlClause havingClauses []sqlClause withClauses []sqlClause + recursiveWith bool err error } -var errSubFilterAlreadySet error = errors.New(`sub-filter already set`) +var errSubFilterAlreadySet = errors.New(`sub-filter already set`) // sub-filter operator values var ( @@ -188,6 +192,17 @@ func (f *filterBuilder) addWith(sql string, args ...interface{}) { f.withClauses = append(f.withClauses, makeClause(sql, args...)) } +// addRecursiveWith adds a with clause and arguments to the filter, and sets it to recursive +//nolint:unused +func (f *filterBuilder) addRecursiveWith(sql string, args ...interface{}) { + if sql == "" { + return + } + + f.addWith(sql, args...) + f.recursiveWith = true +} + func (f *filterBuilder) getSubFilterClause(clause, subFilterClause string) string { ret := clause @@ -389,14 +404,6 @@ func boolCriterionHandler(c *bool, column string) criterionHandlerFunc { } } -func stringLiteralCriterionHandler(v *string, column string) criterionHandlerFunc { - return func(f *filterBuilder) { - if v != nil { - f.addWhere(column+" = ?", v) - } - } -} - // handle for MultiCriterion where there is a join table between the new // objects type joinedMultiCriterionHandlerBuilder struct { @@ -441,8 +448,8 @@ func (m *joinedMultiCriterionHandlerBuilder) handler(criterion *models.MultiCrit } else if criterion.Modifier == models.CriterionModifierExcludes { // excludes all of the provided ids // need to use actual join table name for this - // not exists (select . from where . = .id and . in ) - whereClause = fmt.Sprintf("not exists (select %[1]s.%[2]s from %[1]s where %[1]s.%[2]s = %[3]s.id and %[1]s.%[4]s in %[5]s)", m.joinTable, m.primaryFK, m.primaryTable, m.foreignFK, getInBinding(len(criterion.Value))) + // .id NOT IN (select . from where . in ) + whereClause = fmt.Sprintf("%[1]s.id NOT IN (SELECT %[3]s.%[2]s from %[3]s where %[3]s.%[4]s in %[5]s)", m.primaryTable, m.primaryFK, m.joinTable, m.foreignFK, getInBinding(len(criterion.Value))) } f.addWhere(whereClause, args...) @@ -518,67 +525,157 @@ func (m *stringListCriterionHandlerBuilder) handler(criterion *models.StringCrit } type hierarchicalMultiCriterionHandlerBuilder struct { + tx dbi + primaryTable string foreignTable string foreignFK string - derivedTable string - parentFK string + derivedTable string + parentFK string + relationsTable string } -func addHierarchicalWithClause(f *filterBuilder, value []string, derivedTable, table, parentFK string, depth int) { +func getHierarchicalValues(tx dbi, values []string, table, relationsTable, parentFK string, depth *int) string { var args []interface{} - for _, value := range value { + depthVal := 0 + if depth != nil { + depthVal = *depth + } + + if depthVal == 0 { + valid := true + var valuesClauses []string + for _, value := range values { + id, err := strconv.Atoi(value) + // In case of invalid value just run the query. + // Building VALUES() based on provided values just saves a query when depth is 0. + if err != nil { + valid = false + break + } + + valuesClauses = append(valuesClauses, fmt.Sprintf("(%d,%d)", id, id)) + } + + if valid { + return "VALUES" + strings.Join(valuesClauses, ",") + } + } + + for _, value := range values { args = append(args, value) } inCount := len(args) var depthCondition string - if depth != -1 { - depthCondition = fmt.Sprintf("WHERE depth < %d", depth) + if depthVal != -1 { + depthCondition = fmt.Sprintf("WHERE depth < %d", depthVal) } withClauseMap := utils.StrFormatMap{ - "derivedTable": derivedTable, - "table": table, - "inBinding": getInBinding(inCount), - "parentFK": parentFK, - "depthCondition": depthCondition, - "unionClause": "", + "table": table, + "relationsTable": relationsTable, + "inBinding": getInBinding(inCount), + "recursiveSelect": "", + "parentFK": parentFK, + "depthCondition": depthCondition, + "unionClause": "", } - if depth != 0 { - withClauseMap["unionClause"] = utils.StrFormat(` -UNION SELECT p.id, c.id, depth + 1 FROM {table} as c -INNER JOIN {derivedTable} as p ON c.{parentFK} = p.child_id {depthCondition} + if relationsTable != "" { + withClauseMap["recursiveSelect"] = utils.StrFormat(`SELECT p.root_id, c.child_id, depth + 1 FROM {relationsTable} AS c +INNER JOIN items as p ON c.parent_id = p.item_id +`, withClauseMap) + } else { + withClauseMap["recursiveSelect"] = utils.StrFormat(`SELECT p.root_id, c.id, depth + 1 FROM {table} as c +INNER JOIN items as p ON c.{parentFK} = p.item_id `, withClauseMap) } - withClause := utils.StrFormat(`RECURSIVE {derivedTable} AS ( -SELECT id as id, id as child_id, 0 as depth FROM {table} + if depthVal != 0 { + withClauseMap["unionClause"] = utils.StrFormat(` +UNION {recursiveSelect} {depthCondition} +`, withClauseMap) + } + + withClause := utils.StrFormat(`items AS ( +SELECT id as root_id, id as item_id, 0 as depth FROM {table} WHERE id in {inBinding} {unionClause}) `, withClauseMap) - f.addWith(withClause, args...) + query := fmt.Sprintf("WITH RECURSIVE %s SELECT 'VALUES' || GROUP_CONCAT('(' || root_id || ', ' || item_id || ')') AS val FROM items", withClause) + + var valuesClause string + err := tx.Get(&valuesClause, query, args...) + if err != nil { + logger.Error(err) + // return record which never matches so we don't have to handle error here + return "VALUES(NULL, NULL)" + } + + return valuesClause +} + +func addHierarchicalConditionClauses(f *filterBuilder, criterion *models.HierarchicalMultiCriterionInput, table, idColumn string) { + if criterion.Modifier == models.CriterionModifierIncludes { + f.addWhere(fmt.Sprintf("%s.%s IS NOT NULL", table, idColumn)) + } else if criterion.Modifier == models.CriterionModifierIncludesAll { + f.addWhere(fmt.Sprintf("%s.%s IS NOT NULL", table, idColumn)) + f.addHaving(fmt.Sprintf("count(distinct %s.%s) IS %d", table, idColumn, len(criterion.Value))) + } else if criterion.Modifier == models.CriterionModifierExcludes { + f.addWhere(fmt.Sprintf("%s.%s IS NULL", table, idColumn)) + } } func (m *hierarchicalMultiCriterionHandlerBuilder) handler(criterion *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { return func(f *filterBuilder) { if criterion != nil && len(criterion.Value) > 0 { - addHierarchicalWithClause(f, criterion.Value, m.derivedTable, m.foreignTable, m.parentFK, criterion.Depth) + valuesClause := getHierarchicalValues(m.tx, criterion.Value, m.foreignTable, m.relationsTable, m.parentFK, criterion.Depth) - f.addJoin(m.derivedTable, "", fmt.Sprintf("%s.child_id = %s.%s", m.derivedTable, m.primaryTable, m.foreignFK)) + f.addJoin("(SELECT column1 AS root_id, column2 AS item_id FROM ("+valuesClause+"))", m.derivedTable, fmt.Sprintf("%s.item_id = %s.%s", m.derivedTable, m.primaryTable, m.foreignFK)) - if criterion.Modifier == models.CriterionModifierIncludes { - f.addWhere(fmt.Sprintf("%s.id IS NOT NULL", m.derivedTable)) - } else if criterion.Modifier == models.CriterionModifierIncludesAll { - f.addWhere(fmt.Sprintf("%s.id IS NOT NULL", m.derivedTable)) - f.addHaving(fmt.Sprintf("count(distinct %s.id) IS %d", m.derivedTable, len(criterion.Value))) - } else if criterion.Modifier == models.CriterionModifierExcludes { - f.addWhere(fmt.Sprintf("%s.id IS NULL", m.derivedTable)) - } + addHierarchicalConditionClauses(f, criterion, m.derivedTable, "root_id") + } + } +} + +type joinedHierarchicalMultiCriterionHandlerBuilder struct { + tx dbi + + primaryTable string + foreignTable string + foreignFK string + + parentFK string + relationsTable string + + joinAs string + joinTable string + primaryFK string +} + +func (m *joinedHierarchicalMultiCriterionHandlerBuilder) handler(criterion *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if criterion != nil && len(criterion.Value) > 0 { + valuesClause := getHierarchicalValues(m.tx, criterion.Value, m.foreignTable, m.relationsTable, m.parentFK, criterion.Depth) + + joinAlias := m.joinAs + joinTable := utils.StrFormat(`( + SELECT j.*, d.column1 AS root_id, d.column2 AS item_id FROM {joinTable} AS j + INNER JOIN ({valuesClause}) AS d ON j.{foreignFK} = d.column2 +) +`, utils.StrFormatMap{ + "joinTable": m.joinTable, + "foreignFK": m.foreignFK, + "valuesClause": valuesClause, + }) + + f.addJoin(joinTable, joinAlias, fmt.Sprintf("%s.%s = %s.id", joinAlias, m.primaryFK, m.primaryTable)) + + addHierarchicalConditionClauses(f, criterion, joinAlias, "root_id") } } } diff --git a/pkg/sqlite/gallery.go b/pkg/sqlite/gallery.go index 66bf2032a..2564c068b 100644 --- a/pkg/sqlite/gallery.go +++ b/pkg/sqlite/gallery.go @@ -311,28 +311,18 @@ func galleryIsMissingCriterionHandler(qb *galleryQueryBuilder, isMissing *string } } -func (qb *galleryQueryBuilder) getMultiCriterionHandlerBuilder(foreignTable, joinTable, foreignFK string, addJoinsFunc func(f *filterBuilder)) multiCriterionHandlerBuilder { - return multiCriterionHandlerBuilder{ - primaryTable: galleryTable, - foreignTable: foreignTable, - joinTable: joinTable, - primaryFK: galleryIDColumn, - foreignFK: foreignFK, - addJoinsFunc: addJoinsFunc, - } -} +func galleryTagsCriterionHandler(qb *galleryQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { + h := joinedHierarchicalMultiCriterionHandlerBuilder{ + tx: qb.tx, -func galleryTagsCriterionHandler(qb *galleryQueryBuilder, tags *models.MultiCriterionInput) criterionHandlerFunc { - h := joinedMultiCriterionHandlerBuilder{ primaryTable: galleryTable, - joinTable: galleriesTagsTable, - joinAs: "tags_join", - primaryFK: galleryIDColumn, - foreignFK: tagIDColumn, + foreignTable: tagTable, + foreignFK: "tag_id", - addJoinTable: func(f *filterBuilder) { - qb.tagsRepository().join(f, "tags_join", "galleries.id") - }, + relationsTable: "tags_relations", + joinAs: "image_tag", + joinTable: galleriesTagsTable, + primaryFK: galleryIDColumn, } return h.handler(tags) @@ -386,6 +376,8 @@ func galleryImageCountCriterionHandler(qb *galleryQueryBuilder, imageCount *mode func galleryStudioCriterionHandler(qb *galleryQueryBuilder, studios *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { h := hierarchicalMultiCriterionHandlerBuilder{ + tx: qb.tx, + primaryTable: galleryTable, foreignTable: studioTable, foreignFK: studioIDColumn, @@ -396,31 +388,20 @@ func galleryStudioCriterionHandler(qb *galleryQueryBuilder, studios *models.Hier return h.handler(studios) } -func galleryPerformerTagsCriterionHandler(qb *galleryQueryBuilder, performerTagsFilter *models.MultiCriterionInput) criterionHandlerFunc { +func galleryPerformerTagsCriterionHandler(qb *galleryQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { return func(f *filterBuilder) { - if performerTagsFilter != nil && len(performerTagsFilter.Value) > 0 { - qb.performersRepository().join(f, "performers_join", "galleries.id") - f.addJoin("performers_tags", "performer_tags_join", "performers_join.performer_id = performer_tags_join.performer_id") + if tags != nil && len(tags.Value) > 0 { + valuesClause := getHierarchicalValues(qb.tx, tags.Value, tagTable, "tags_relations", "", tags.Depth) - var args []interface{} - for _, tagID := range performerTagsFilter.Value { - args = append(args, tagID) - } + f.addWith(`performer_tags AS ( +SELECT pg.gallery_id, t.column1 AS root_tag_id FROM performers_galleries pg +INNER JOIN performers_tags pt ON pt.performer_id = pg.performer_id +INNER JOIN (` + valuesClause + `) t ON t.column2 = pt.tag_id +)`) - if performerTagsFilter.Modifier == models.CriterionModifierIncludes { - // includes any of the provided ids - f.addWhere("performer_tags_join.tag_id IN "+getInBinding(len(performerTagsFilter.Value)), args...) - } else if performerTagsFilter.Modifier == models.CriterionModifierIncludesAll { - // includes all of the provided ids - f.addWhere("performer_tags_join.tag_id IN "+getInBinding(len(performerTagsFilter.Value)), args...) - f.addHaving(fmt.Sprintf("count(distinct performer_tags_join.tag_id) IS %d", len(performerTagsFilter.Value))) - } else if performerTagsFilter.Modifier == models.CriterionModifierExcludes { - f.addWhere(fmt.Sprintf(`not exists - (select performers_galleries.performer_id from performers_galleries - left join performers_tags on performers_tags.performer_id = performers_galleries.performer_id where - performers_galleries.gallery_id = galleries.id AND - performers_tags.tag_id in %s)`, getInBinding(len(performerTagsFilter.Value))), args...) - } + f.addJoin("performer_tags", "", "performer_tags.gallery_id = galleries.id") + + addHierarchicalConditionClauses(f, tags, "performer_tags", "root_tag_id") } } } diff --git a/pkg/sqlite/gallery_test.go b/pkg/sqlite/gallery_test.go index 5ba8db446..d07907156 100644 --- a/pkg/sqlite/gallery_test.go +++ b/pkg/sqlite/gallery_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test @@ -620,7 +621,7 @@ func TestGalleryQueryPerformers(t *testing.T) { func TestGalleryQueryTags(t *testing.T) { withTxn(func(r models.Repository) error { sqb := r.Gallery() - tagCriterion := models.MultiCriterionInput{ + tagCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdxWithGallery]), strconv.Itoa(tagIDs[tagIdx1WithGallery]), @@ -640,7 +641,7 @@ func TestGalleryQueryTags(t *testing.T) { assert.True(t, gallery.ID == galleryIDs[galleryIdxWithTag] || gallery.ID == galleryIDs[galleryIdxWithTwoTags]) } - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithGallery]), strconv.Itoa(tagIDs[tagIdx2WithGallery]), @@ -653,7 +654,7 @@ func TestGalleryQueryTags(t *testing.T) { assert.Len(t, galleries, 1) assert.Equal(t, galleryIDs[galleryIdxWithTwoTags], galleries[0].ID) - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithGallery]), }, @@ -680,7 +681,6 @@ func TestGalleryQueryStudio(t *testing.T) { strconv.Itoa(studioIDs[studioIdxWithGallery]), }, Modifier: models.CriterionModifierIncludes, - Depth: 0, } galleryFilter := models.GalleryFilterType{ @@ -699,7 +699,6 @@ func TestGalleryQueryStudio(t *testing.T) { strconv.Itoa(studioIDs[studioIdxWithGallery]), }, Modifier: models.CriterionModifierExcludes, - Depth: 0, } q := getGalleryStringValue(galleryIdxWithStudio, titleField) @@ -717,12 +716,13 @@ func TestGalleryQueryStudio(t *testing.T) { func TestGalleryQueryStudioDepth(t *testing.T) { withTxn(func(r models.Repository) error { sqb := r.Gallery() + depth := 2 studioCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(studioIDs[studioIdxWithGrandChild]), }, Modifier: models.CriterionModifierIncludes, - Depth: 2, + Depth: &depth, } galleryFilter := models.GalleryFilterType{ @@ -732,7 +732,7 @@ func TestGalleryQueryStudioDepth(t *testing.T) { galleries := queryGallery(t, sqb, &galleryFilter, nil) assert.Len(t, galleries, 1) - studioCriterion.Depth = 1 + depth = 1 galleries = queryGallery(t, sqb, &galleryFilter, nil) assert.Len(t, galleries, 0) @@ -744,12 +744,14 @@ func TestGalleryQueryStudioDepth(t *testing.T) { // ensure id is correct assert.Equal(t, galleryIDs[galleryIdxWithGrandChildStudio], galleries[0].ID) + depth = 2 + studioCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(studioIDs[studioIdxWithGrandChild]), }, Modifier: models.CriterionModifierExcludes, - Depth: 2, + Depth: &depth, } q := getGalleryStringValue(galleryIdxWithGrandChildStudio, pathField) @@ -760,7 +762,7 @@ func TestGalleryQueryStudioDepth(t *testing.T) { galleries = queryGallery(t, sqb, &galleryFilter, &findFilter) assert.Len(t, galleries, 0) - studioCriterion.Depth = 1 + depth = 1 galleries = queryGallery(t, sqb, &galleryFilter, &findFilter) assert.Len(t, galleries, 1) @@ -775,7 +777,7 @@ func TestGalleryQueryStudioDepth(t *testing.T) { func TestGalleryQueryPerformerTags(t *testing.T) { withTxn(func(r models.Repository) error { sqb := r.Gallery() - tagCriterion := models.MultiCriterionInput{ + tagCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdxWithPerformer]), strconv.Itoa(tagIDs[tagIdx1WithPerformer]), @@ -795,7 +797,7 @@ func TestGalleryQueryPerformerTags(t *testing.T) { assert.True(t, gallery.ID == galleryIDs[galleryIdxWithPerformerTag] || gallery.ID == galleryIDs[galleryIdxWithPerformerTwoTags]) } - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithPerformer]), strconv.Itoa(tagIDs[tagIdx2WithPerformer]), @@ -808,7 +810,7 @@ func TestGalleryQueryPerformerTags(t *testing.T) { assert.Len(t, galleries, 1) assert.Equal(t, galleryIDs[galleryIdxWithPerformerTwoTags], galleries[0].ID) - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithPerformer]), }, diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index d0b6f16f8..01598c51a 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -348,17 +348,18 @@ func (qb *imageQueryBuilder) getMultiCriterionHandlerBuilder(foreignTable, joinT } } -func imageTagsCriterionHandler(qb *imageQueryBuilder, tags *models.MultiCriterionInput) criterionHandlerFunc { - h := joinedMultiCriterionHandlerBuilder{ - primaryTable: imageTable, - joinTable: imagesTagsTable, - joinAs: "tags_join", - primaryFK: imageIDColumn, - foreignFK: tagIDColumn, +func imageTagsCriterionHandler(qb *imageQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { + h := joinedHierarchicalMultiCriterionHandlerBuilder{ + tx: qb.tx, - addJoinTable: func(f *filterBuilder) { - qb.tagsRepository().join(f, "tags_join", "images.id") - }, + primaryTable: imageTable, + foreignTable: tagTable, + foreignFK: "tag_id", + + relationsTable: "tags_relations", + joinAs: "image_tag", + joinTable: imagesTagsTable, + primaryFK: imageIDColumn, } return h.handler(tags) @@ -412,6 +413,8 @@ func imagePerformerCountCriterionHandler(qb *imageQueryBuilder, performerCount * func imageStudioCriterionHandler(qb *imageQueryBuilder, studios *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { h := hierarchicalMultiCriterionHandlerBuilder{ + tx: qb.tx, + primaryTable: imageTable, foreignTable: studioTable, foreignFK: studioIDColumn, @@ -422,31 +425,20 @@ func imageStudioCriterionHandler(qb *imageQueryBuilder, studios *models.Hierarch return h.handler(studios) } -func imagePerformerTagsCriterionHandler(qb *imageQueryBuilder, performerTagsFilter *models.MultiCriterionInput) criterionHandlerFunc { +func imagePerformerTagsCriterionHandler(qb *imageQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { return func(f *filterBuilder) { - if performerTagsFilter != nil && len(performerTagsFilter.Value) > 0 { - qb.performersRepository().join(f, "performers_join", "images.id") - f.addJoin("performers_tags", "performer_tags_join", "performers_join.performer_id = performer_tags_join.performer_id") + if tags != nil && len(tags.Value) > 0 { + valuesClause := getHierarchicalValues(qb.tx, tags.Value, tagTable, "tags_relations", "", tags.Depth) - var args []interface{} - for _, tagID := range performerTagsFilter.Value { - args = append(args, tagID) - } + f.addWith(`performer_tags AS ( +SELECT pi.image_id, t.column1 AS root_tag_id FROM performers_images pi +INNER JOIN performers_tags pt ON pt.performer_id = pi.performer_id +INNER JOIN (` + valuesClause + `) t ON t.column2 = pt.tag_id +)`) - if performerTagsFilter.Modifier == models.CriterionModifierIncludes { - // includes any of the provided ids - f.addWhere("performer_tags_join.tag_id IN "+getInBinding(len(performerTagsFilter.Value)), args...) - } else if performerTagsFilter.Modifier == models.CriterionModifierIncludesAll { - // includes all of the provided ids - f.addWhere("performer_tags_join.tag_id IN "+getInBinding(len(performerTagsFilter.Value)), args...) - f.addHaving(fmt.Sprintf("count(distinct performer_tags_join.tag_id) IS %d", len(performerTagsFilter.Value))) - } else if performerTagsFilter.Modifier == models.CriterionModifierExcludes { - f.addWhere(fmt.Sprintf(`not exists - (select performers_images.performer_id from performers_images - left join performers_tags on performers_tags.performer_id = performers_images.performer_id where - performers_images.image_id = images.id AND - performers_tags.tag_id in %s)`, getInBinding(len(performerTagsFilter.Value))), args...) - } + f.addJoin("performer_tags", "", "performer_tags.image_id = images.id") + + addHierarchicalConditionClauses(f, tags, "performer_tags", "root_tag_id") } } } diff --git a/pkg/sqlite/image_test.go b/pkg/sqlite/image_test.go index 50006685b..59802e0d8 100644 --- a/pkg/sqlite/image_test.go +++ b/pkg/sqlite/image_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test @@ -721,7 +722,7 @@ func TestImageQueryPerformers(t *testing.T) { func TestImageQueryTags(t *testing.T) { withTxn(func(r models.Repository) error { sqb := r.Image() - tagCriterion := models.MultiCriterionInput{ + tagCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdxWithImage]), strconv.Itoa(tagIDs[tagIdx1WithImage]), @@ -745,7 +746,7 @@ func TestImageQueryTags(t *testing.T) { assert.True(t, image.ID == imageIDs[imageIdxWithTag] || image.ID == imageIDs[imageIdxWithTwoTags]) } - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithImage]), strconv.Itoa(tagIDs[tagIdx2WithImage]), @@ -761,7 +762,7 @@ func TestImageQueryTags(t *testing.T) { assert.Len(t, images, 1) assert.Equal(t, imageIDs[imageIdxWithTwoTags], images[0].ID) - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithImage]), }, @@ -791,7 +792,6 @@ func TestImageQueryStudio(t *testing.T) { strconv.Itoa(studioIDs[studioIdxWithImage]), }, Modifier: models.CriterionModifierIncludes, - Depth: 0, } imageFilter := models.ImageFilterType{ @@ -813,7 +813,6 @@ func TestImageQueryStudio(t *testing.T) { strconv.Itoa(studioIDs[studioIdxWithImage]), }, Modifier: models.CriterionModifierExcludes, - Depth: 0, } q := getImageStringValue(imageIdxWithStudio, titleField) @@ -834,12 +833,13 @@ func TestImageQueryStudio(t *testing.T) { func TestImageQueryStudioDepth(t *testing.T) { withTxn(func(r models.Repository) error { sqb := r.Image() + depth := 2 studioCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(studioIDs[studioIdxWithGrandChild]), }, Modifier: models.CriterionModifierIncludes, - Depth: 2, + Depth: &depth, } imageFilter := models.ImageFilterType{ @@ -849,7 +849,7 @@ func TestImageQueryStudioDepth(t *testing.T) { images := queryImages(t, sqb, &imageFilter, nil) assert.Len(t, images, 1) - studioCriterion.Depth = 1 + depth = 1 images = queryImages(t, sqb, &imageFilter, nil) assert.Len(t, images, 0) @@ -861,12 +861,14 @@ func TestImageQueryStudioDepth(t *testing.T) { // ensure id is correct assert.Equal(t, imageIDs[imageIdxWithGrandChildStudio], images[0].ID) + depth = 2 + studioCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(studioIDs[studioIdxWithGrandChild]), }, Modifier: models.CriterionModifierExcludes, - Depth: 2, + Depth: &depth, } q := getImageStringValue(imageIdxWithGrandChildStudio, titleField) @@ -877,7 +879,7 @@ func TestImageQueryStudioDepth(t *testing.T) { images = queryImages(t, sqb, &imageFilter, &findFilter) assert.Len(t, images, 0) - studioCriterion.Depth = 1 + depth = 1 images = queryImages(t, sqb, &imageFilter, &findFilter) assert.Len(t, images, 1) @@ -901,7 +903,7 @@ func queryImages(t *testing.T, sqb models.ImageReader, imageFilter *models.Image func TestImageQueryPerformerTags(t *testing.T) { withTxn(func(r models.Repository) error { sqb := r.Image() - tagCriterion := models.MultiCriterionInput{ + tagCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdxWithPerformer]), strconv.Itoa(tagIDs[tagIdx1WithPerformer]), @@ -921,7 +923,7 @@ func TestImageQueryPerformerTags(t *testing.T) { assert.True(t, image.ID == imageIDs[imageIdxWithPerformerTag] || image.ID == imageIDs[imageIdxWithPerformerTwoTags]) } - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithPerformer]), strconv.Itoa(tagIDs[tagIdx2WithPerformer]), @@ -934,7 +936,7 @@ func TestImageQueryPerformerTags(t *testing.T) { assert.Len(t, images, 1) assert.Equal(t, imageIDs[imageIdxWithPerformerTwoTags], images[0].ID) - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithPerformer]), }, diff --git a/pkg/sqlite/movies.go b/pkg/sqlite/movies.go index 49d970459..40340ac13 100644 --- a/pkg/sqlite/movies.go +++ b/pkg/sqlite/movies.go @@ -126,6 +126,7 @@ func (qb *movieQueryBuilder) makeFilter(movieFilter *models.MovieFilterType) *fi query.handleCriterion(movieIsMissingCriterionHandler(qb, movieFilter.IsMissing)) query.handleCriterion(stringCriterionHandler(movieFilter.URL, "movies.url")) query.handleCriterion(movieStudioCriterionHandler(qb, movieFilter.Studios)) + query.handleCriterion(moviePerformersCriterionHandler(qb, movieFilter.Performers)) return query } @@ -194,6 +195,8 @@ func movieIsMissingCriterionHandler(qb *movieQueryBuilder, isMissing *string) cr func movieStudioCriterionHandler(qb *movieQueryBuilder, studios *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { h := hierarchicalMultiCriterionHandlerBuilder{ + tx: qb.tx, + primaryTable: movieTable, foreignTable: studioTable, foreignFK: studioIDColumn, @@ -204,6 +207,35 @@ func movieStudioCriterionHandler(qb *movieQueryBuilder, studios *models.Hierarch return h.handler(studios) } +func moviePerformersCriterionHandler(qb *movieQueryBuilder, performers *models.MultiCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if performers != nil && len(performers.Value) > 0 { + var args []interface{} + for _, arg := range performers.Value { + args = append(args, arg) + } + + // Hack, can't apply args to join, nor inner join on a left join, so use CTE instead + f.addWith(`movies_performers AS ( + SELECT movies_scenes.movie_id, performers_scenes.performer_id + FROM movies_scenes + INNER JOIN performers_scenes ON movies_scenes.scene_id = performers_scenes.scene_id + WHERE performers_scenes.performer_id IN`+getInBinding(len(performers.Value))+` + )`, args...) + f.addJoin("movies_performers", "", "movies.id = movies_performers.movie_id") + + if performers.Modifier == models.CriterionModifierIncludes { + f.addWhere("movies_performers.performer_id IS NOT NULL") + } else if performers.Modifier == models.CriterionModifierIncludesAll { + f.addWhere("movies_performers.performer_id IS NOT NULL") + f.addHaving("COUNT(DISTINCT movies_performers.performer_id) = ?", len(performers.Value)) + } else if performers.Modifier == models.CriterionModifierExcludes { + f.addWhere("movies_performers.performer_id IS NULL") + } + } + } +} + func (qb *movieQueryBuilder) getMovieSort(findFilter *models.FindFilterType) string { var sort string var direction string @@ -276,3 +308,42 @@ func (qb *movieQueryBuilder) GetBackImage(movieID int) ([]byte, error) { query := `SELECT back_image from movies_images WHERE movie_id = ?` return getImage(qb.tx, query, movieID) } + +func (qb *movieQueryBuilder) FindByPerformerID(performerID int) ([]*models.Movie, error) { + query := `SELECT DISTINCT movies.* +FROM movies +INNER JOIN movies_scenes ON movies.id = movies_scenes.movie_id +INNER JOIN performers_scenes ON performers_scenes.scene_id = movies_scenes.scene_id +WHERE performers_scenes.performer_id = ? +` + args := []interface{}{performerID} + return qb.queryMovies(query, args) +} + +func (qb *movieQueryBuilder) CountByPerformerID(performerID int) (int, error) { + query := `SELECT COUNT(DISTINCT movies_scenes.movie_id) AS count +FROM movies_scenes +INNER JOIN performers_scenes ON performers_scenes.scene_id = movies_scenes.scene_id +WHERE performers_scenes.performer_id = ? +` + args := []interface{}{performerID} + return qb.runCountQuery(query, args) +} + +func (qb *movieQueryBuilder) FindByStudioID(studioID int) ([]*models.Movie, error) { + query := `SELECT movies.* +FROM movies +WHERE movies.studio_id = ? +` + args := []interface{}{studioID} + return qb.queryMovies(query, args) +} + +func (qb *movieQueryBuilder) CountByStudioID(studioID int) (int, error) { + query := `SELECT COUNT(1) AS count +FROM movies +WHERE movies.studio_id = ? +` + args := []interface{}{studioID} + return qb.runCountQuery(query, args) +} diff --git a/pkg/sqlite/movies_test.go b/pkg/sqlite/movies_test.go index d067f96d8..681c9fa09 100644 --- a/pkg/sqlite/movies_test.go +++ b/pkg/sqlite/movies_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test @@ -81,7 +82,6 @@ func TestMovieQueryStudio(t *testing.T) { strconv.Itoa(studioIDs[studioIdxWithMovie]), }, Modifier: models.CriterionModifierIncludes, - Depth: 0, } movieFilter := models.MovieFilterType{ @@ -103,7 +103,6 @@ func TestMovieQueryStudio(t *testing.T) { strconv.Itoa(studioIDs[studioIdxWithMovie]), }, Modifier: models.CriterionModifierExcludes, - Depth: 0, } q := getMovieStringValue(movieIdxWithStudio, titleField) diff --git a/pkg/sqlite/performer.go b/pkg/sqlite/performer.go index 526a6e360..c8b3f86de 100644 --- a/pkg/sqlite/performer.go +++ b/pkg/sqlite/performer.go @@ -280,7 +280,7 @@ func (qb *performerQueryBuilder) makeFilter(filter *models.PerformerFilterType) query.handleCriterion(performerTagsCriterionHandler(qb, filter.Tags)) - query.handleCriterion(performerStudiosCriterionHandler(filter.Studios)) + query.handleCriterion(performerStudiosCriterionHandler(qb, filter.Studios)) query.handleCriterion(performerTagCountCriterionHandler(qb, filter.TagCount)) query.handleCriterion(performerSceneCountCriterionHandler(qb, filter.SceneCount)) @@ -376,17 +376,18 @@ func performerAgeFilterCriterionHandler(age *models.IntCriterionInput) criterion } } -func performerTagsCriterionHandler(qb *performerQueryBuilder, tags *models.MultiCriterionInput) criterionHandlerFunc { - h := joinedMultiCriterionHandlerBuilder{ - primaryTable: performerTable, - joinTable: performersTagsTable, - joinAs: "tags_join", - primaryFK: performerIDColumn, - foreignFK: tagIDColumn, +func performerTagsCriterionHandler(qb *performerQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { + h := joinedHierarchicalMultiCriterionHandlerBuilder{ + tx: qb.tx, - addJoinTable: func(f *filterBuilder) { - qb.tagsRepository().join(f, "tags_join", "performers.id") - }, + primaryTable: performerTable, + foreignTable: tagTable, + foreignFK: "tag_id", + + relationsTable: "tags_relations", + joinAs: "image_tag", + joinTable: performersTagsTable, + primaryFK: performerIDColumn, } return h.handler(tags) @@ -432,7 +433,7 @@ func performerGalleryCountCriterionHandler(qb *performerQueryBuilder, count *mod return h.handler(count) } -func performerStudiosCriterionHandler(studios *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { +func performerStudiosCriterionHandler(qb *performerQueryBuilder, studios *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { return func(f *filterBuilder) { if studios != nil { var clauseCondition string @@ -465,13 +466,13 @@ func performerStudiosCriterionHandler(studios *models.HierarchicalMultiCriterion }, } - const derivedStudioTable = "studio" const derivedPerformerStudioTable = "performer_studio" - addHierarchicalWithClause(f, studios.Value, derivedStudioTable, studioTable, "parent_id", studios.Depth) + valuesClause := getHierarchicalValues(qb.tx, studios.Value, studioTable, "", "parent_id", studios.Depth) + f.addWith("studio(root_id, item_id) AS (" + valuesClause + ")") templStr := `SELECT performer_id FROM {primaryTable} INNER JOIN {joinTable} ON {primaryTable}.id = {joinTable}.{primaryFK} - INNER JOIN studio ON {primaryTable}.studio_id = studio.child_id` + INNER JOIN studio ON {primaryTable}.studio_id = studio.item_id` var unions []string for _, c := range formatMaps { diff --git a/pkg/sqlite/performer_test.go b/pkg/sqlite/performer_test.go index c5292f384..829016d57 100644 --- a/pkg/sqlite/performer_test.go +++ b/pkg/sqlite/performer_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test @@ -164,7 +165,7 @@ func TestPerformerQueryEthnicityAndRating(t *testing.T) { }) } -func TestPerformerQueryPathNotRating(t *testing.T) { +func TestPerformerQueryEthnicityNotRating(t *testing.T) { const performerIdx = 1 performerRating := getRating(performerIdx) @@ -499,7 +500,7 @@ func queryPerformers(t *testing.T, qb models.PerformerReader, performerFilter *m func TestPerformerQueryTags(t *testing.T) { withTxn(func(r models.Repository) error { sqb := r.Performer() - tagCriterion := models.MultiCriterionInput{ + tagCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdxWithPerformer]), strconv.Itoa(tagIDs[tagIdx1WithPerformer]), @@ -518,7 +519,7 @@ func TestPerformerQueryTags(t *testing.T) { assert.True(t, performer.ID == performerIDs[performerIdxWithTag] || performer.ID == performerIDs[performerIdxWithTwoTags]) } - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithPerformer]), strconv.Itoa(tagIDs[tagIdx2WithPerformer]), @@ -531,7 +532,7 @@ func TestPerformerQueryTags(t *testing.T) { assert.Len(t, performers, 1) assert.Equal(t, sceneIDs[performerIdxWithTwoTags], performers[0].ID) - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithPerformer]), }, diff --git a/pkg/sqlite/query.go b/pkg/sqlite/query.go index 2d31ee583..59b200641 100644 --- a/pkg/sqlite/query.go +++ b/pkg/sqlite/query.go @@ -1,11 +1,7 @@ package sqlite import ( - "fmt" - "regexp" "strings" - - "github.com/stashapp/stash/pkg/models" ) type queryBuilder struct { @@ -18,6 +14,7 @@ type queryBuilder struct { havingClauses []string args []interface{} withClauses []string + recursiveWith bool sortAndPagination string @@ -32,7 +29,7 @@ func (qb queryBuilder) executeFind() ([]int, int, error) { body := qb.body body += qb.joins.toSQL() - return qb.repository.executeFindQuery(body, qb.args, qb.sortAndPagination, qb.whereClauses, qb.havingClauses, qb.withClauses) + return qb.repository.executeFindQuery(body, qb.args, qb.sortAndPagination, qb.whereClauses, qb.havingClauses, qb.withClauses, qb.recursiveWith) } func (qb queryBuilder) executeCount() (int, error) { @@ -45,7 +42,11 @@ func (qb queryBuilder) executeCount() (int, error) { withClause := "" if len(qb.withClauses) > 0 { - withClause = "WITH " + strings.Join(qb.withClauses, ", ") + " " + var recursive string + if qb.recursiveWith { + recursive = " RECURSIVE " + } + withClause = "WITH " + recursive + strings.Join(qb.withClauses, ", ") + " " } body = qb.repository.buildQueryBody(body, qb.whereClauses, qb.havingClauses) @@ -69,12 +70,14 @@ func (qb *queryBuilder) addHaving(clauses ...string) { } } -func (qb *queryBuilder) addWith(clauses ...string) { +func (qb *queryBuilder) addWith(recursive bool, clauses ...string) { for _, clause := range clauses { if len(clause) > 0 { qb.withClauses = append(qb.withClauses, clause) } } + + qb.recursiveWith = qb.recursiveWith || recursive } func (qb *queryBuilder) addArg(args ...interface{}) { @@ -104,7 +107,7 @@ func (qb *queryBuilder) addFilter(f *filterBuilder) { clause, args := f.generateWithClauses() if len(clause) > 0 { - qb.addWith(clause) + qb.addWith(f.recursiveWith, clause) } if len(args) > 0 { @@ -132,67 +135,3 @@ func (qb *queryBuilder) addFilter(f *filterBuilder) { qb.addJoins(f.getAllJoins()...) } - -func (qb *queryBuilder) handleIntCriterionInput(c *models.IntCriterionInput, column string) { - if c != nil { - clause, args := getIntCriterionWhereClause(column, *c) - qb.addWhere(clause) - qb.addArg(args...) - } -} - -func (qb *queryBuilder) handleStringCriterionInput(c *models.StringCriterionInput, column string) { - if c != nil { - if modifier := c.Modifier; c.Modifier.IsValid() { - switch modifier { - case models.CriterionModifierIncludes: - clause, thisArgs := getSearchBinding([]string{column}, c.Value, false) - qb.addWhere(clause) - qb.addArg(thisArgs...) - case models.CriterionModifierExcludes: - clause, thisArgs := getSearchBinding([]string{column}, c.Value, true) - qb.addWhere(clause) - qb.addArg(thisArgs...) - case models.CriterionModifierEquals: - qb.addWhere(column + " LIKE ?") - qb.addArg(c.Value) - case models.CriterionModifierNotEquals: - qb.addWhere(column + " NOT LIKE ?") - qb.addArg(c.Value) - case models.CriterionModifierMatchesRegex: - if _, err := regexp.Compile(c.Value); err != nil { - qb.err = err - return - } - qb.addWhere(fmt.Sprintf("(%s IS NOT NULL AND %[1]s regexp ?)", column)) - qb.addArg(c.Value) - case models.CriterionModifierNotMatchesRegex: - if _, err := regexp.Compile(c.Value); err != nil { - qb.err = err - return - } - qb.addWhere(fmt.Sprintf("(%s IS NULL OR %[1]s NOT regexp ?)", column)) - qb.addArg(c.Value) - case models.CriterionModifierIsNull: - qb.addWhere("(" + column + " IS NULL OR TRIM(" + column + ") = '')") - case models.CriterionModifierNotNull: - qb.addWhere("(" + column + " IS NOT NULL AND TRIM(" + column + ") != '')") - default: - clause, count := getSimpleCriterionClause(modifier, "?") - qb.addWhere(column + " " + clause) - if count == 1 { - qb.addArg(c.Value) - } - } - } - } -} - -func (qb *queryBuilder) handleCountCriterion(countFilter *models.IntCriterionInput, primaryTable, joinTable, primaryFK string) { - if countFilter != nil { - clause, args := getCountCriterionClause(primaryTable, joinTable, primaryFK, *countFilter) - - qb.addWhere(clause) - qb.addArg(args...) - } -} diff --git a/pkg/sqlite/repository.go b/pkg/sqlite/repository.go index 182d0223f..b6b107151 100644 --- a/pkg/sqlite/repository.go +++ b/pkg/sqlite/repository.go @@ -238,20 +238,24 @@ func (r *repository) buildQueryBody(body string, whereClauses []string, havingCl if len(whereClauses) > 0 { body = body + " WHERE " + strings.Join(whereClauses, " AND ") // TODO handle AND or OR } - body = body + " GROUP BY " + r.tableName + ".id " if len(havingClauses) > 0 { + body = body + " GROUP BY " + r.tableName + ".id " body = body + " HAVING " + strings.Join(havingClauses, " AND ") // TODO handle AND or OR } return body } -func (r *repository) executeFindQuery(body string, args []interface{}, sortAndPagination string, whereClauses []string, havingClauses []string, withClauses []string) ([]int, int, error) { +func (r *repository) executeFindQuery(body string, args []interface{}, sortAndPagination string, whereClauses []string, havingClauses []string, withClauses []string, recursiveWith bool) ([]int, int, error) { body = r.buildQueryBody(body, whereClauses, havingClauses) withClause := "" if len(withClauses) > 0 { - withClause = "WITH " + strings.Join(withClauses, ", ") + " " + var recursive string + if recursiveWith { + recursive = " RECURSIVE " + } + withClause = "WITH " + recursive + strings.Join(withClauses, ", ") + " " } countQuery := withClause + r.buildCountQuery(body) @@ -269,10 +273,10 @@ func (r *repository) executeFindQuery(body string, args []interface{}, sortAndPa idsResult, idsErr = r.runIdsQuery(idsQuery, args) if countErr != nil { - return nil, 0, fmt.Errorf("Error executing count query with SQL: %s, args: %v, error: %s", countQuery, args, countErr.Error()) + return nil, 0, fmt.Errorf("error executing count query with SQL: %s, args: %v, error: %s", countQuery, args, countErr.Error()) } if idsErr != nil { - return nil, 0, fmt.Errorf("Error executing find query with SQL: %s, args: %v, error: %s", idsQuery, args, idsErr.Error()) + return nil, 0, fmt.Errorf("error executing find query with SQL: %s, args: %v, error: %s", idsQuery, args, idsErr.Error()) } return idsResult, countResult, nil diff --git a/pkg/sqlite/saved_filter_test.go b/pkg/sqlite/saved_filter_test.go index 4cd33c97a..5ec049290 100644 --- a/pkg/sqlite/saved_filter_test.go +++ b/pkg/sqlite/saved_filter_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index 92cdd5c2c..5aa0d4722 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -548,12 +548,19 @@ func (qb *sceneQueryBuilder) getMultiCriterionHandlerBuilder(foreignTable, joinT } } -func sceneTagsCriterionHandler(qb *sceneQueryBuilder, tags *models.MultiCriterionInput) criterionHandlerFunc { - addJoinsFunc := func(f *filterBuilder) { - qb.tagsRepository().join(f, "tags_join", "scenes.id") - f.addJoin("tags", "", "tags_join.tag_id = tags.id") +func sceneTagsCriterionHandler(qb *sceneQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { + h := joinedHierarchicalMultiCriterionHandlerBuilder{ + tx: qb.tx, + + primaryTable: sceneTable, + foreignTable: tagTable, + foreignFK: "tag_id", + + relationsTable: "tags_relations", + joinAs: "scene_tag", + joinTable: scenesTagsTable, + primaryFK: sceneIDColumn, } - h := qb.getMultiCriterionHandlerBuilder(tagTable, scenesTagsTable, tagIDColumn, addJoinsFunc) return h.handler(tags) } @@ -596,6 +603,8 @@ func scenePerformerCountCriterionHandler(qb *sceneQueryBuilder, performerCount * func sceneStudioCriterionHandler(qb *sceneQueryBuilder, studios *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { h := hierarchicalMultiCriterionHandlerBuilder{ + tx: qb.tx, + primaryTable: sceneTable, foreignTable: studioTable, foreignFK: studioIDColumn, @@ -615,31 +624,20 @@ func sceneMoviesCriterionHandler(qb *sceneQueryBuilder, movies *models.MultiCrit return h.handler(movies) } -func scenePerformerTagsCriterionHandler(qb *sceneQueryBuilder, performerTagsFilter *models.MultiCriterionInput) criterionHandlerFunc { +func scenePerformerTagsCriterionHandler(qb *sceneQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { return func(f *filterBuilder) { - if performerTagsFilter != nil && len(performerTagsFilter.Value) > 0 { - qb.performersRepository().join(f, "performers_join", "scenes.id") - f.addJoin("performers_tags", "performer_tags_join", "performers_join.performer_id = performer_tags_join.performer_id") + if tags != nil && len(tags.Value) > 0 { + valuesClause := getHierarchicalValues(qb.tx, tags.Value, tagTable, "tags_relations", "", tags.Depth) - var args []interface{} - for _, tagID := range performerTagsFilter.Value { - args = append(args, tagID) - } + f.addWith(`performer_tags AS ( +SELECT ps.scene_id, t.column1 AS root_tag_id FROM performers_scenes ps +INNER JOIN performers_tags pt ON pt.performer_id = ps.performer_id +INNER JOIN (` + valuesClause + `) t ON t.column2 = pt.tag_id +)`) - if performerTagsFilter.Modifier == models.CriterionModifierIncludes { - // includes any of the provided ids - f.addWhere("performer_tags_join.tag_id IN "+getInBinding(len(performerTagsFilter.Value)), args...) - } else if performerTagsFilter.Modifier == models.CriterionModifierIncludesAll { - // includes all of the provided ids - f.addWhere("performer_tags_join.tag_id IN "+getInBinding(len(performerTagsFilter.Value)), args...) - f.addHaving(fmt.Sprintf("count(distinct performer_tags_join.tag_id) IS %d", len(performerTagsFilter.Value))) - } else if performerTagsFilter.Modifier == models.CriterionModifierExcludes { - f.addWhere(fmt.Sprintf(`not exists - (select performers_scenes.performer_id from performers_scenes - left join performers_tags on performers_tags.performer_id = performers_scenes.performer_id where - performers_scenes.scene_id = scenes.id AND - performers_tags.tag_id in %s)`, getInBinding(len(performerTagsFilter.Value))), args...) - } + f.addJoin("performer_tags", "", "performer_tags.scene_id = scenes.id") + + addHierarchicalConditionClauses(f, tags, "performer_tags", "root_tag_id") } } } diff --git a/pkg/sqlite/scene_marker.go b/pkg/sqlite/scene_marker.go index 0ad3cda43..7872b21c5 100644 --- a/pkg/sqlite/scene_marker.go +++ b/pkg/sqlite/scene_marker.go @@ -3,7 +3,6 @@ package sqlite import ( "database/sql" "fmt" - "strconv" "github.com/stashapp/stash/pkg/database" "github.com/stashapp/stash/pkg/models" @@ -127,6 +126,17 @@ func (qb *sceneMarkerQueryBuilder) Wall(q *string) ([]*models.SceneMarker, error return qb.querySceneMarkers(query, nil) } +func (qb *sceneMarkerQueryBuilder) makeFilter(sceneMarkerFilter *models.SceneMarkerFilterType) *filterBuilder { + query := &filterBuilder{} + + query.handleCriterion(sceneMarkerTagIDCriterionHandler(qb, sceneMarkerFilter.TagID)) + query.handleCriterion(sceneMarkerTagsCriterionHandler(qb, sceneMarkerFilter.Tags)) + query.handleCriterion(sceneMarkerSceneTagsCriterionHandler(qb, sceneMarkerFilter.SceneTags)) + query.handleCriterion(sceneMarkerPerformersCriterionHandler(qb, sceneMarkerFilter.Performers)) + + return query +} + func (qb *sceneMarkerQueryBuilder) Query(sceneMarkerFilter *models.SceneMarkerFilterType, findFilter *models.FindFilterType) ([]*models.SceneMarker, int, error) { if sceneMarkerFilter == nil { sceneMarkerFilter = &models.SceneMarkerFilterType{} @@ -135,121 +145,23 @@ func (qb *sceneMarkerQueryBuilder) Query(sceneMarkerFilter *models.SceneMarkerFi findFilter = &models.FindFilterType{} } - var whereClauses []string - var havingClauses []string - var args []interface{} - body := selectDistinctIDs("scene_markers") - body = body + ` - left join tags as primary_tag on primary_tag.id = scene_markers.primary_tag_id - left join scenes as scene on scene.id = scene_markers.scene_id - left join scene_markers_tags as tags_join on tags_join.scene_marker_id = scene_markers.id - left join tags on tags_join.tag_id = tags.id - ` + query := qb.newQuery() - if tagsFilter := sceneMarkerFilter.Tags; tagsFilter != nil && len(tagsFilter.Value) > 0 { - //select `scene_markers`.* from `scene_markers` - //left join `tags` as `primary_tags_join` - // on `primary_tags_join`.`id` = `scene_markers`.`primary_tag_id` - // and `primary_tags_join`.`id` in ('3', '37', '9', '89') - //left join `scene_markers_tags` as `tags_join` - // on `tags_join`.`scene_marker_id` = `scene_markers`.`id` - // and `tags_join`.`tag_id` in ('3', '37', '9', '89') - //group by `scene_markers`.`id` - //having ((count(distinct `primary_tags_join`.`id`) + count(distinct `tags_join`.`tag_id`)) = 4) - - length := len(tagsFilter.Value) - - if tagsFilter.Modifier == models.CriterionModifierIncludes || tagsFilter.Modifier == models.CriterionModifierIncludesAll { - body += " LEFT JOIN tags AS ptj ON ptj.id = scene_markers.primary_tag_id AND ptj.id IN " + getInBinding(length) - body += " LEFT JOIN scene_markers_tags AS tj ON tj.scene_marker_id = scene_markers.id AND tj.tag_id IN " + getInBinding(length) - - // only one required for include any - requiredCount := 1 - - // all required for include all - if tagsFilter.Modifier == models.CriterionModifierIncludesAll { - requiredCount = length - } - - havingClauses = append(havingClauses, "((COUNT(DISTINCT ptj.id) + COUNT(DISTINCT tj.tag_id)) >= "+strconv.Itoa(requiredCount)+")") - } else if tagsFilter.Modifier == models.CriterionModifierExcludes { - // excludes all of the provided ids - whereClauses = append(whereClauses, "scene_markers.primary_tag_id not in "+getInBinding(length)) - whereClauses = append(whereClauses, "not exists (select smt.scene_marker_id from scene_markers_tags as smt where smt.scene_marker_id = scene_markers.id and smt.tag_id in "+getInBinding(length)+")") - } - - for _, tagID := range tagsFilter.Value { - args = append(args, tagID) - } - for _, tagID := range tagsFilter.Value { - args = append(args, tagID) - } - } - - if sceneTagsFilter := sceneMarkerFilter.SceneTags; sceneTagsFilter != nil && len(sceneTagsFilter.Value) > 0 { - length := len(sceneTagsFilter.Value) - - if sceneTagsFilter.Modifier == models.CriterionModifierIncludes || sceneTagsFilter.Modifier == models.CriterionModifierIncludesAll { - body += " LEFT JOIN scenes_tags AS scene_tags_join ON scene_tags_join.scene_id = scene.id AND scene_tags_join.tag_id IN " + getInBinding(length) - - // only one required for include any - requiredCount := 1 - - // all required for include all - if sceneTagsFilter.Modifier == models.CriterionModifierIncludesAll { - requiredCount = length - } - - havingClauses = append(havingClauses, "COUNT(DISTINCT scene_tags_join.tag_id) >= "+strconv.Itoa(requiredCount)) - } else if sceneTagsFilter.Modifier == models.CriterionModifierExcludes { - // excludes all of the provided ids - whereClauses = append(whereClauses, "not exists (select st.scene_id from scenes_tags as st where st.scene_id = scene.id AND st.tag_id IN "+getInBinding(length)+")") - } - - for _, tagID := range sceneTagsFilter.Value { - args = append(args, tagID) - } - } - - if performersFilter := sceneMarkerFilter.Performers; performersFilter != nil && len(performersFilter.Value) > 0 { - length := len(performersFilter.Value) - - if performersFilter.Modifier == models.CriterionModifierIncludes || performersFilter.Modifier == models.CriterionModifierIncludesAll { - body += " LEFT JOIN performers_scenes as scene_performers ON scene.id = scene_performers.scene_id" - whereClauses = append(whereClauses, "scene_performers.performer_id IN "+getInBinding(length)) - - // only one required for include any - requiredCount := 1 - - // all required for include all - if performersFilter.Modifier == models.CriterionModifierIncludesAll { - requiredCount = length - } - - havingClauses = append(havingClauses, "COUNT(DISTINCT scene_performers.performer_id) >= "+strconv.Itoa(requiredCount)) - } else if performersFilter.Modifier == models.CriterionModifierExcludes { - // excludes all of the provided ids - whereClauses = append(whereClauses, "not exists (select sp.scene_id from performers_scenes as sp where sp.scene_id = scene.id AND sp.performer_id IN "+getInBinding(length)+")") - } - - for _, performerID := range performersFilter.Value { - args = append(args, performerID) - } - } + query.body = selectDistinctIDs("scene_markers") if q := findFilter.Q; q != nil && *q != "" { - searchColumns := []string{"scene_markers.title", "scene.title"} + searchColumns := []string{"scene_markers.title", "scenes.title"} clause, thisArgs := getSearchBinding(searchColumns, *q, false) - whereClauses = append(whereClauses, clause) - args = append(args, thisArgs...) + query.addWhere(clause) + query.addArg(thisArgs...) } - if tagID := sceneMarkerFilter.TagID; tagID != nil { - whereClauses = append(whereClauses, "(scene_markers.primary_tag_id = "+*tagID+" OR tags.id = "+*tagID+")") - } + filter := qb.makeFilter(sceneMarkerFilter) - sortAndPagination := qb.getSceneMarkerSort(findFilter) + getPagination(findFilter) - idsResult, countResult, err := qb.executeFindQuery(body, args, sortAndPagination, whereClauses, havingClauses, []string{}) + query.addFilter(filter) + + query.sortAndPagination = qb.getSceneMarkerSort(&query, findFilter) + getPagination(findFilter) + idsResult, countResult, err := query.executeFind() if err != nil { return nil, 0, err } @@ -267,13 +179,83 @@ func (qb *sceneMarkerQueryBuilder) Query(sceneMarkerFilter *models.SceneMarkerFi return sceneMarkers, countResult, nil } -func (qb *sceneMarkerQueryBuilder) getSceneMarkerSort(findFilter *models.FindFilterType) string { +func sceneMarkerTagIDCriterionHandler(qb *sceneMarkerQueryBuilder, tagID *string) criterionHandlerFunc { + return func(f *filterBuilder) { + if tagID != nil { + f.addJoin("scene_markers_tags", "", "scene_markers_tags.scene_marker_id = scene_markers.id") + + f.addWhere("(scene_markers.primary_tag_id = ? OR scene_markers_tags.tag_id = ?)", *tagID, *tagID) + } + } +} + +func sceneMarkerTagsCriterionHandler(qb *sceneMarkerQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if tags != nil && len(tags.Value) > 0 { + valuesClause := getHierarchicalValues(qb.tx, tags.Value, tagTable, "tags_relations", "", tags.Depth) + + f.addWith(`marker_tags AS ( +SELECT mt.scene_marker_id, t.column1 AS root_tag_id FROM scene_markers_tags mt +INNER JOIN (` + valuesClause + `) t ON t.column2 = mt.tag_id +UNION +SELECT m.id, t.column1 FROM scene_markers m +INNER JOIN (` + valuesClause + `) t ON t.column2 = m.primary_tag_id +)`) + + f.addJoin("marker_tags", "", "marker_tags.scene_marker_id = scene_markers.id") + + addHierarchicalConditionClauses(f, tags, "marker_tags", "root_tag_id") + } + } +} + +func sceneMarkerSceneTagsCriterionHandler(qb *sceneMarkerQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if tags != nil && len(tags.Value) > 0 { + valuesClause := getHierarchicalValues(qb.tx, tags.Value, tagTable, "tags_relations", "", tags.Depth) + + f.addWith(`scene_tags AS ( +SELECT st.scene_id, t.column1 AS root_tag_id FROM scenes_tags st +INNER JOIN (` + valuesClause + `) t ON t.column2 = st.tag_id +)`) + + f.addJoin("scene_tags", "", "scene_tags.scene_id = scene_markers.scene_id") + + addHierarchicalConditionClauses(f, tags, "scene_tags", "root_tag_id") + } + } +} + +func sceneMarkerPerformersCriterionHandler(qb *sceneMarkerQueryBuilder, performers *models.MultiCriterionInput) criterionHandlerFunc { + h := joinedMultiCriterionHandlerBuilder{ + primaryTable: sceneTable, + joinTable: performersScenesTable, + joinAs: "performers_join", + primaryFK: sceneIDColumn, + foreignFK: performerIDColumn, + + addJoinTable: func(f *filterBuilder) { + f.addJoin(performersScenesTable, "performers_join", "performers_join.scene_id = scene_markers.scene_id") + }, + } + + handler := h.handler(performers) + return func(f *filterBuilder) { + // Make sure scenes is included, otherwise excludes filter fails + f.addJoin(sceneTable, "", "scenes.id = scene_markers.scene_id") + handler(f) + } +} + +func (qb *sceneMarkerQueryBuilder) getSceneMarkerSort(query *queryBuilder, findFilter *models.FindFilterType) string { sort := findFilter.GetSort("title") direction := findFilter.GetDirection() tableName := "scene_markers" if sort == "scenes_updated_at" { + // ensure scene table is joined + query.join(sceneTable, "", "scenes.id = scene_markers.scene_id") sort = "updated_at" - tableName = "scene" + tableName = "scenes" } return getSort(sort, direction, tableName) } diff --git a/pkg/sqlite/scene_marker_test.go b/pkg/sqlite/scene_marker_test.go index bc3d95daa..d50c181de 100644 --- a/pkg/sqlite/scene_marker_test.go +++ b/pkg/sqlite/scene_marker_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test @@ -67,6 +68,21 @@ func TestMarkerCountByTagID(t *testing.T) { }) } +func TestMarkerQuerySortBySceneUpdated(t *testing.T) { + withTxn(func(r models.Repository) error { + sort := "scenes_updated_at" + _, _, err := r.SceneMarker().Query(nil, &models.FindFilterType{ + Sort: &sort, + }) + + if err != nil { + t.Errorf("Error querying scene markers: %s", err.Error()) + } + + return nil + }) +} + // TODO Update // TODO Destroy // TODO Find diff --git a/pkg/sqlite/scene_test.go b/pkg/sqlite/scene_test.go index ec95447da..48790d1ae 100644 --- a/pkg/sqlite/scene_test.go +++ b/pkg/sqlite/scene_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test @@ -1033,7 +1034,7 @@ func TestSceneQueryPerformers(t *testing.T) { func TestSceneQueryTags(t *testing.T) { withTxn(func(r models.Repository) error { sqb := r.Scene() - tagCriterion := models.MultiCriterionInput{ + tagCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdxWithScene]), strconv.Itoa(tagIDs[tagIdx1WithScene]), @@ -1053,7 +1054,7 @@ func TestSceneQueryTags(t *testing.T) { assert.True(t, scene.ID == sceneIDs[sceneIdxWithTag] || scene.ID == sceneIDs[sceneIdxWithTwoTags]) } - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithScene]), strconv.Itoa(tagIDs[tagIdx2WithScene]), @@ -1066,7 +1067,7 @@ func TestSceneQueryTags(t *testing.T) { assert.Len(t, scenes, 1) assert.Equal(t, sceneIDs[sceneIdxWithTwoTags], scenes[0].ID) - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithScene]), }, @@ -1088,7 +1089,7 @@ func TestSceneQueryTags(t *testing.T) { func TestSceneQueryPerformerTags(t *testing.T) { withTxn(func(r models.Repository) error { sqb := r.Scene() - tagCriterion := models.MultiCriterionInput{ + tagCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdxWithPerformer]), strconv.Itoa(tagIDs[tagIdx1WithPerformer]), @@ -1108,7 +1109,7 @@ func TestSceneQueryPerformerTags(t *testing.T) { assert.True(t, scene.ID == sceneIDs[sceneIdxWithPerformerTag] || scene.ID == sceneIDs[sceneIdxWithPerformerTwoTags]) } - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithPerformer]), strconv.Itoa(tagIDs[tagIdx2WithPerformer]), @@ -1121,7 +1122,7 @@ func TestSceneQueryPerformerTags(t *testing.T) { assert.Len(t, scenes, 1) assert.Equal(t, sceneIDs[sceneIdxWithPerformerTwoTags], scenes[0].ID) - tagCriterion = models.MultiCriterionInput{ + tagCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdx1WithPerformer]), }, @@ -1148,7 +1149,6 @@ func TestSceneQueryStudio(t *testing.T) { strconv.Itoa(studioIDs[studioIdxWithScene]), }, Modifier: models.CriterionModifierIncludes, - Depth: 0, } sceneFilter := models.SceneFilterType{ @@ -1167,7 +1167,6 @@ func TestSceneQueryStudio(t *testing.T) { strconv.Itoa(studioIDs[studioIdxWithScene]), }, Modifier: models.CriterionModifierExcludes, - Depth: 0, } q := getSceneStringValue(sceneIdxWithStudio, titleField) @@ -1185,12 +1184,13 @@ func TestSceneQueryStudio(t *testing.T) { func TestSceneQueryStudioDepth(t *testing.T) { withTxn(func(r models.Repository) error { sqb := r.Scene() + depth := 2 studioCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(studioIDs[studioIdxWithGrandChild]), }, Modifier: models.CriterionModifierIncludes, - Depth: 2, + Depth: &depth, } sceneFilter := models.SceneFilterType{ @@ -1200,7 +1200,7 @@ func TestSceneQueryStudioDepth(t *testing.T) { scenes := queryScene(t, sqb, &sceneFilter, nil) assert.Len(t, scenes, 1) - studioCriterion.Depth = 1 + depth = 1 scenes = queryScene(t, sqb, &sceneFilter, nil) assert.Len(t, scenes, 0) @@ -1211,13 +1211,14 @@ func TestSceneQueryStudioDepth(t *testing.T) { // ensure id is correct assert.Equal(t, sceneIDs[sceneIdxWithGrandChildStudio], scenes[0].ID) + depth = 2 studioCriterion = models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(studioIDs[studioIdxWithGrandChild]), }, Modifier: models.CriterionModifierExcludes, - Depth: 2, + Depth: &depth, } q := getSceneStringValue(sceneIdxWithGrandChildStudio, titleField) @@ -1228,7 +1229,7 @@ func TestSceneQueryStudioDepth(t *testing.T) { scenes = queryScene(t, sqb, &sceneFilter, &findFilter) assert.Len(t, scenes, 0) - studioCriterion.Depth = 1 + depth = 1 scenes = queryScene(t, sqb, &sceneFilter, &findFilter) assert.Len(t, scenes, 1) diff --git a/pkg/sqlite/scraped_item.go b/pkg/sqlite/scraped_item.go index 39e040f89..30f772dc9 100644 --- a/pkg/sqlite/scraped_item.go +++ b/pkg/sqlite/scraped_item.go @@ -72,14 +72,6 @@ func (qb *scrapedItemQueryBuilder) getScrapedItemsSort(findFilter *models.FindFi return getSort(sort, direction, "scraped_items") } -func (qb *scrapedItemQueryBuilder) queryScrapedItem(query string, args []interface{}) (*models.ScrapedItem, error) { - results, err := qb.queryScrapedItems(query, args) - if err != nil || len(results) < 1 { - return nil, err - } - return results[0], nil -} - func (qb *scrapedItemQueryBuilder) queryScrapedItems(query string, args []interface{}) ([]*models.ScrapedItem, error) { var ret models.ScrapedItems if err := qb.query(query, args, &ret); err != nil { diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index d25e3c4e4..116d8eabb 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test @@ -7,7 +8,6 @@ import ( "database/sql" "errors" "fmt" - "io/ioutil" "os" "strconv" "testing" @@ -151,6 +151,11 @@ const ( tagIdxWithGallery tagIdx1WithGallery tagIdx2WithGallery + tagIdxWithChildTag + tagIdxWithParentTag + tagIdxWithGrandChild + tagIdxWithParentAndChild + tagIdxWithGrandParent // new indexes above // tags with dup names start from the end tagIdx1WithDupName @@ -345,6 +350,14 @@ var ( } ) +var ( + tagParentLinks = [][2]int{ + {tagIdxWithChildTag, tagIdxWithParentTag}, + {tagIdxWithGrandChild, tagIdxWithParentAndChild}, + {tagIdxWithParentAndChild, tagIdxWithGrandParent}, + } +) + func TestMain(m *testing.M) { ret := runTests(m) os.Exit(ret) @@ -380,7 +393,7 @@ func testTeardown(databaseFile string) { func runTests(m *testing.M) int { // create the database file - f, err := ioutil.TempFile("", "*.sqlite") + f, err := os.CreateTemp("", "*.sqlite") if err != nil { panic(fmt.Sprintf("Could not create temporary file: %s", err.Error())) } @@ -499,6 +512,10 @@ func populateDB() error { return fmt.Errorf("error linking gallery studios: %s", err.Error()) } + if err := linkTagsParent(r.Tag()); err != nil { + return fmt.Errorf("error linking tags parent: %s", err.Error()) + } + if err := createMarker(r.SceneMarker(), sceneIdxWithMarker, tagIdxWithPrimaryMarker, []int{tagIdxWithMarker}); err != nil { return fmt.Errorf("error creating scene marker: %s", err.Error()) } @@ -865,6 +882,22 @@ func getTagPerformerCount(id int) int { return 0 } +func getTagParentCount(id int) int { + if id == tagIDs[tagIdxWithParentTag] || id == tagIDs[tagIdxWithGrandParent] || id == tagIDs[tagIdxWithParentAndChild] { + return 1 + } + + return 0 +} + +func getTagChildCount(id int) int { + if id == tagIDs[tagIdxWithChildTag] || id == tagIDs[tagIdxWithGrandChild] || id == tagIDs[tagIdxWithParentAndChild] { + return 1 + } + + return 0 +} + //createTags creates n tags with plain Name and o tags with camel cased NaMe included func createTags(tqb models.TagReaderWriter, n int, o int) error { const namePlain = "Name" @@ -962,6 +995,12 @@ func createStudios(sqb models.StudioReaderWriter, n int, o int) error { return err } + // add alias + alias := getStudioStringValue(i, "Alias") + if err := sqb.UpdateAliases(created.ID, []string{alias}); err != nil { + return fmt.Errorf("error setting studio alias: %s", err.Error()) + } + studioIDs = append(studioIDs, created.ID) studioNames = append(studioNames, created.Name.String) } @@ -1225,6 +1264,25 @@ func linkStudiosParent(qb models.StudioWriter) error { }) } +func linkTagsParent(qb models.TagReaderWriter) error { + return doLinks(tagParentLinks, func(parentIndex, childIndex int) error { + tagID := tagIDs[childIndex] + parentTags, err := qb.FindByChildTagID(tagID) + if err != nil { + return err + } + + var parentIDs []int + for _, parentTag := range parentTags { + parentIDs = append(parentIDs, parentTag.ID) + } + + parentIDs = append(parentIDs, tagIDs[parentIndex]) + + return qb.UpdateParentTags(tagID, parentIDs) + }) +} + func addTagImage(qb models.TagWriter, tagIndex int) error { return qb.UpdateImage(tagIDs[tagIndex], models.DefaultTagImage) } diff --git a/pkg/sqlite/sql.go b/pkg/sqlite/sql.go index b3bcdd514..827bdbda2 100644 --- a/pkg/sqlite/sql.go +++ b/pkg/sqlite/sql.go @@ -8,7 +8,6 @@ import ( "strconv" "strings" - "github.com/jmoiron/sqlx" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" ) @@ -148,32 +147,6 @@ func getInBinding(length int) string { return "(" + bindings + ")" } -func getCriterionModifierBinding(criterionModifier models.CriterionModifier, value interface{}) (string, int) { - var length int - switch x := value.(type) { - case []string: - length = len(x) - case []int: - length = len(x) - default: - length = 1 - } - if modifier := criterionModifier.String(); criterionModifier.IsValid() { - switch modifier { - case "EQUALS", "NOT_EQUALS", "GREATER_THAN", "LESS_THAN", "IS_NULL", "NOT_NULL", "BETWEEN", "NOT_BETWEEN": - return getSimpleCriterionClause(criterionModifier, "?") - case "INCLUDES": - return "IN " + getInBinding(length), length // TODO? - case "EXCLUDES": - return "NOT IN " + getInBinding(length), length // TODO? - default: - logger.Errorf("todo") - return "= ?", 1 // TODO - } - } - return "= ?", 1 // TODO -} - func getSimpleCriterionClause(criterionModifier models.CriterionModifier, rhs string) (string, int) { if modifier := criterionModifier.String(); criterionModifier.IsValid() { switch modifier { @@ -209,20 +182,16 @@ func getIntCriterionWhereClause(column string, input models.IntCriterionInput) ( switch input.Modifier { case "EQUALS", "NOT_EQUALS": args = []interface{}{input.Value} - break case "LESS_THAN": args = []interface{}{input.Value} - break case "GREATER_THAN": args = []interface{}{input.Value} - break case "BETWEEN", "NOT_BETWEEN": upper := 0 if input.Value2 != nil { upper = *input.Value2 } args = []interface{}{input.Value, upper} - break } return column + " " + binding, args @@ -242,7 +211,7 @@ func getMultiCriterionClause(primaryTable, foreignTable, joinTable, primaryFK, f } else if criterion.Modifier == models.CriterionModifierExcludes { // excludes all of the provided ids if joinTable != "" { - whereClause = "not exists (select " + joinTable + "." + primaryFK + " from " + joinTable + " where " + joinTable + "." + primaryFK + " = " + primaryTable + ".id and " + joinTable + "." + foreignFK + " in " + getInBinding(len(criterion.Value)) + ")" + whereClause = primaryTable + ".id not in (select " + joinTable + "." + primaryFK + " from " + joinTable + " where " + joinTable + "." + foreignFK + " in " + getInBinding(len(criterion.Value)) + ")" } else { whereClause = "not exists (select s.id from " + primaryTable + " as s where s.id = " + primaryTable + ".id and s." + foreignFK + " in " + getInBinding(len(criterion.Value)) + ")" } @@ -256,12 +225,6 @@ func getCountCriterionClause(primaryTable, joinTable, primaryFK string, criterio return getIntCriterionWhereClause(lhs, criterion) } -func ensureTx(tx *sqlx.Tx) { - if tx == nil { - panic("must use a transaction") - } -} - func getImage(tx dbi, query string, args ...interface{}) ([]byte, error) { rows, err := tx.Queryx(query, args...) diff --git a/pkg/sqlite/stash_id_test.go b/pkg/sqlite/stash_id_test.go index 5c9b44f4c..0f57bef19 100644 --- a/pkg/sqlite/stash_id_test.go +++ b/pkg/sqlite/stash_id_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test diff --git a/pkg/sqlite/studio.go b/pkg/sqlite/studio.go index 34d657148..746a89b12 100644 --- a/pkg/sqlite/studio.go +++ b/pkg/sqlite/studio.go @@ -10,6 +10,8 @@ import ( const studioTable = "studios" const studioIDColumn = "studio_id" +const studioAliasesTable = "studio_aliases" +const studioAliasColumn = "alias" type studioQueryBuilder struct { repository @@ -126,19 +128,91 @@ func (qb *studioQueryBuilder) QueryForAutoTag(words []string) ([]*models.Studio, // TODO - Query needs to be changed to support queries of this type, and // this method should be removed query := selectAll(studioTable) + query += " LEFT JOIN studio_aliases ON studio_aliases.studio_id = studios.id" var whereClauses []string var args []interface{} for _, w := range words { - whereClauses = append(whereClauses, "name like ?") - args = append(args, w+"%") + ww := w + "%" + whereClauses = append(whereClauses, "studios.name like ?") + args = append(args, ww) + + // include aliases + whereClauses = append(whereClauses, "studio_aliases.alias like ?") + args = append(args, ww) } where := strings.Join(whereClauses, " OR ") return qb.queryStudios(query+" WHERE "+where, args) } +func (qb *studioQueryBuilder) validateFilter(filter *models.StudioFilterType) error { + const and = "AND" + const or = "OR" + const not = "NOT" + + if filter.And != nil { + if filter.Or != nil { + return illegalFilterCombination(and, or) + } + if filter.Not != nil { + return illegalFilterCombination(and, not) + } + + return qb.validateFilter(filter.And) + } + + if filter.Or != nil { + if filter.Not != nil { + return illegalFilterCombination(or, not) + } + + return qb.validateFilter(filter.Or) + } + + if filter.Not != nil { + return qb.validateFilter(filter.Not) + } + + return nil +} + +func (qb *studioQueryBuilder) makeFilter(studioFilter *models.StudioFilterType) *filterBuilder { + query := &filterBuilder{} + + if studioFilter.And != nil { + query.and(qb.makeFilter(studioFilter.And)) + } + if studioFilter.Or != nil { + query.or(qb.makeFilter(studioFilter.Or)) + } + if studioFilter.Not != nil { + query.not(qb.makeFilter(studioFilter.Not)) + } + + query.handleCriterion(stringCriterionHandler(studioFilter.Name, studioTable+".name")) + query.handleCriterion(stringCriterionHandler(studioFilter.Details, studioTable+".details")) + query.handleCriterion(stringCriterionHandler(studioFilter.URL, studioTable+".url")) + query.handleCriterion(intCriterionHandler(studioFilter.Rating, studioTable+".rating")) + + query.handleCriterion(criterionHandlerFunc(func(f *filterBuilder) { + if studioFilter.StashID != nil { + qb.stashIDRepository().join(f, "studio_stash_ids", "studios.id") + stringCriterionHandler(studioFilter.StashID, "studio_stash_ids.stash_id")(f) + } + })) + + query.handleCriterion(studioIsMissingCriterionHandler(qb, studioFilter.IsMissing)) + query.handleCriterion(studioSceneCountCriterionHandler(qb, studioFilter.SceneCount)) + query.handleCriterion(studioImageCountCriterionHandler(qb, studioFilter.ImageCount)) + query.handleCriterion(studioGalleryCountCriterionHandler(qb, studioFilter.GalleryCount)) + query.handleCriterion(studioParentCriterionHandler(qb, studioFilter.Parents)) + query.handleCriterion(studioAliasCriterionHandler(qb, studioFilter.Aliases)) + + return query +} + func (qb *studioQueryBuilder) Query(studioFilter *models.StudioFilterType, findFilter *models.FindFilterType) ([]*models.Studio, int, error) { if studioFilter == nil { studioFilter = &models.StudioFilterType{} @@ -150,57 +224,22 @@ func (qb *studioQueryBuilder) Query(studioFilter *models.StudioFilterType, findF query := qb.newQuery() query.body = selectDistinctIDs("studios") - query.body += ` - left join scenes on studios.id = scenes.studio_id - left join studio_stash_ids on studio_stash_ids.studio_id = studios.id - ` if q := findFilter.Q; q != nil && *q != "" { - searchColumns := []string{"studios.name"} + query.join(studioAliasesTable, "", "studio_aliases.studio_id = studios.id") + searchColumns := []string{"studios.name", "studio_aliases.alias"} clause, thisArgs := getSearchBinding(searchColumns, *q, false) query.addWhere(clause) query.addArg(thisArgs...) } - if parentsFilter := studioFilter.Parents; parentsFilter != nil && len(parentsFilter.Value) > 0 { - query.body += ` - left join studios as parent_studio on parent_studio.id = studios.parent_id - ` - - for _, studioID := range parentsFilter.Value { - query.addArg(studioID) - } - - whereClause, havingClause := getMultiCriterionClause("studios", "parent_studio", "", "", "parent_id", parentsFilter) - - query.addWhere(whereClause) - query.addHaving(havingClause) + if err := qb.validateFilter(studioFilter); err != nil { + return nil, 0, err } + filter := qb.makeFilter(studioFilter) - if rating := studioFilter.Rating; rating != nil { - query.handleIntCriterionInput(studioFilter.Rating, "studios.rating") - } - query.handleCountCriterion(studioFilter.SceneCount, studioTable, sceneTable, studioIDColumn) - query.handleCountCriterion(studioFilter.ImageCount, studioTable, imageTable, studioIDColumn) - query.handleCountCriterion(studioFilter.GalleryCount, studioTable, galleryTable, studioIDColumn) - query.handleStringCriterionInput(studioFilter.Name, "studios.name") - query.handleStringCriterionInput(studioFilter.Details, "studios.details") - query.handleStringCriterionInput(studioFilter.URL, "studios.url") - query.handleStringCriterionInput(studioFilter.StashID, "studio_stash_ids.stash_id") - - if isMissingFilter := studioFilter.IsMissing; isMissingFilter != nil && *isMissingFilter != "" { - switch *isMissingFilter { - case "image": - query.body += `left join studios_image on studios_image.studio_id = studios.id - ` - query.addWhere("studios_image.studio_id IS NULL") - case "stash_id": - query.addWhere("studio_stash_ids.studio_id IS NULL") - default: - query.addWhere("studios." + *isMissingFilter + " IS NULL") - } - } + query.addFilter(filter) query.sortAndPagination = qb.getStudioSort(findFilter) + getPagination(findFilter) idsResult, countResult, err := query.executeFind() @@ -221,6 +260,83 @@ func (qb *studioQueryBuilder) Query(studioFilter *models.StudioFilterType, findF return studios, countResult, nil } +func studioIsMissingCriterionHandler(qb *studioQueryBuilder, isMissing *string) criterionHandlerFunc { + return func(f *filterBuilder) { + if isMissing != nil && *isMissing != "" { + switch *isMissing { + case "image": + f.addJoin("studios_image", "", "studios_image.studio_id = studios.id") + f.addWhere("studios_image.studio_id IS NULL") + case "stash_id": + qb.stashIDRepository().join(f, "studio_stash_ids", "studios.id") + f.addWhere("studio_stash_ids.studio_id IS NULL") + default: + f.addWhere("(studios." + *isMissing + " IS NULL OR TRIM(studios." + *isMissing + ") = '')") + } + } + } +} + +func studioSceneCountCriterionHandler(qb *studioQueryBuilder, sceneCount *models.IntCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if sceneCount != nil { + f.addJoin("scenes", "", "scenes.studio_id = studios.id") + clause, args := getIntCriterionWhereClause("count(distinct scenes.id)", *sceneCount) + + f.addHaving(clause, args...) + } + } +} + +func studioImageCountCriterionHandler(qb *studioQueryBuilder, imageCount *models.IntCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if imageCount != nil { + f.addJoin("images", "", "images.studio_id = studios.id") + clause, args := getIntCriterionWhereClause("count(distinct images.id)", *imageCount) + + f.addHaving(clause, args...) + } + } +} + +func studioGalleryCountCriterionHandler(qb *studioQueryBuilder, galleryCount *models.IntCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if galleryCount != nil { + f.addJoin("galleries", "", "galleries.studio_id = studios.id") + clause, args := getIntCriterionWhereClause("count(distinct galleries.id)", *galleryCount) + + f.addHaving(clause, args...) + } + } +} + +func studioParentCriterionHandler(qb *studioQueryBuilder, parents *models.MultiCriterionInput) criterionHandlerFunc { + addJoinsFunc := func(f *filterBuilder) { + f.addJoin("studios", "parent_studio", "parent_studio.id = studios.parent_id") + } + h := multiCriterionHandlerBuilder{ + primaryTable: studioTable, + foreignTable: "parent_studio", + joinTable: "", + primaryFK: studioIDColumn, + foreignFK: "parent_id", + addJoinsFunc: addJoinsFunc, + } + return h.handler(parents) +} + +func studioAliasCriterionHandler(qb *studioQueryBuilder, alias *models.StringCriterionInput) criterionHandlerFunc { + h := stringListCriterionHandlerBuilder{ + joinTable: studioAliasesTable, + stringColumn: studioAliasColumn, + addJoinTable: func(f *filterBuilder) { + qb.aliasRepository().join(f, "", "studios.id") + }, + } + + return h.handler(alias) +} + func (qb *studioQueryBuilder) getStudioSort(findFilter *models.FindFilterType) string { var sort string var direction string @@ -233,6 +349,8 @@ func (qb *studioQueryBuilder) getStudioSort(findFilter *models.FindFilterType) s } switch sort { + case "scenes_count": + return getCountSort(studioTable, sceneTable, studioIDColumn, direction) case "images_count": return getCountSort(studioTable, imageTable, studioIDColumn, direction) case "galleries_count": @@ -303,3 +421,22 @@ func (qb *studioQueryBuilder) GetStashIDs(studioID int) ([]*models.StashID, erro func (qb *studioQueryBuilder) UpdateStashIDs(studioID int, stashIDs []models.StashID) error { return qb.stashIDRepository().replace(studioID, stashIDs) } + +func (qb *studioQueryBuilder) aliasRepository() *stringRepository { + return &stringRepository{ + repository: repository{ + tx: qb.tx, + tableName: studioAliasesTable, + idColumn: studioIDColumn, + }, + stringColumn: studioAliasColumn, + } +} + +func (qb *studioQueryBuilder) GetAliases(studioID int) ([]string, error) { + return qb.aliasRepository().get(studioID) +} + +func (qb *studioQueryBuilder) UpdateAliases(studioID int, aliases []string) error { + return qb.aliasRepository().replace(studioID, aliases) +} diff --git a/pkg/sqlite/studio_test.go b/pkg/sqlite/studio_test.go index 8c97e50e7..8e623e53f 100644 --- a/pkg/sqlite/studio_test.go +++ b/pkg/sqlite/studio_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test @@ -45,6 +46,143 @@ func TestStudioFindByName(t *testing.T) { }) } +func TestStudioQueryNameOr(t *testing.T) { + const studio1Idx = 1 + const studio2Idx = 2 + + studio1Name := getStudioStringValue(studio1Idx, "Name") + studio2Name := getStudioStringValue(studio2Idx, "Name") + + studioFilter := models.StudioFilterType{ + Name: &models.StringCriterionInput{ + Value: studio1Name, + Modifier: models.CriterionModifierEquals, + }, + Or: &models.StudioFilterType{ + Name: &models.StringCriterionInput{ + Value: studio2Name, + Modifier: models.CriterionModifierEquals, + }, + }, + } + + withTxn(func(r models.Repository) error { + sqb := r.Studio() + + studios := queryStudio(t, sqb, &studioFilter, nil) + + assert.Len(t, studios, 2) + assert.Equal(t, studio1Name, studios[0].Name.String) + assert.Equal(t, studio2Name, studios[1].Name.String) + + return nil + }) +} + +func TestStudioQueryNameAndUrl(t *testing.T) { + const studioIdx = 1 + studioName := getStudioStringValue(studioIdx, "Name") + studioUrl := getStudioNullStringValue(studioIdx, urlField) + + studioFilter := models.StudioFilterType{ + Name: &models.StringCriterionInput{ + Value: studioName, + Modifier: models.CriterionModifierEquals, + }, + And: &models.StudioFilterType{ + URL: &models.StringCriterionInput{ + Value: studioUrl.String, + Modifier: models.CriterionModifierEquals, + }, + }, + } + + withTxn(func(r models.Repository) error { + sqb := r.Studio() + + studios := queryStudio(t, sqb, &studioFilter, nil) + + assert.Len(t, studios, 1) + assert.Equal(t, studioName, studios[0].Name.String) + assert.Equal(t, studioUrl.String, studios[0].URL.String) + + return nil + }) +} + +func TestStudioQueryNameNotUrl(t *testing.T) { + const studioIdx = 1 + + studioUrl := getStudioNullStringValue(studioIdx, urlField) + + nameCriterion := models.StringCriterionInput{ + Value: "studio_.*1_Name", + Modifier: models.CriterionModifierMatchesRegex, + } + + urlCriterion := models.StringCriterionInput{ + Value: studioUrl.String, + Modifier: models.CriterionModifierEquals, + } + + studioFilter := models.StudioFilterType{ + Name: &nameCriterion, + Not: &models.StudioFilterType{ + URL: &urlCriterion, + }, + } + + withTxn(func(r models.Repository) error { + sqb := r.Studio() + + studios := queryStudio(t, sqb, &studioFilter, nil) + + for _, studio := range studios { + verifyString(t, studio.Name.String, nameCriterion) + urlCriterion.Modifier = models.CriterionModifierNotEquals + verifyNullString(t, studio.URL, urlCriterion) + } + + return nil + }) +} + +func TestStudioIllegalQuery(t *testing.T) { + assert := assert.New(t) + + const studioIdx = 1 + subFilter := models.StudioFilterType{ + Name: &models.StringCriterionInput{ + Value: getStudioStringValue(studioIdx, "Name"), + Modifier: models.CriterionModifierEquals, + }, + } + + studioFilter := &models.StudioFilterType{ + And: &subFilter, + Or: &subFilter, + } + + withTxn(func(r models.Repository) error { + sqb := r.Studio() + + _, _, err := sqb.Query(studioFilter, nil) + assert.NotNil(err) + + studioFilter.Or = nil + studioFilter.Not = &subFilter + _, _, err = sqb.Query(studioFilter, nil) + assert.NotNil(err) + + studioFilter.And = nil + studioFilter.Or = &subFilter + _, _, err = sqb.Query(studioFilter, nil) + assert.NotNil(err) + + return nil + }) +} + func TestStudioQueryForAutoTag(t *testing.T) { withTxn(func(r models.Repository) error { tqb := r.Studio() @@ -61,6 +199,17 @@ func TestStudioQueryForAutoTag(t *testing.T) { assert.Equal(t, strings.ToLower(studioNames[studioIdxWithScene]), strings.ToLower(studios[0].Name.String)) assert.Equal(t, strings.ToLower(studioNames[studioIdxWithScene]), strings.ToLower(studios[1].Name.String)) + // find by alias + name = getStudioStringValue(studioIdxWithScene, "Alias") + studios, err = tqb.QueryForAutoTag([]string{name}) + + if err != nil { + t.Errorf("Error finding studios: %s", err.Error()) + } + + assert.Len(t, studios, 1) + assert.Equal(t, studioIDs[studioIdxWithScene], studios[0].ID) + return nil }) } @@ -362,7 +511,6 @@ func verifyStudiosImageCount(t *testing.T, imageCountCriterion models.IntCriteri Studios: &models.HierarchicalMultiCriterionInput{ Value: []string{strconv.Itoa(studio.ID)}, Modifier: models.CriterionModifierIncludes, - Depth: 0, }, }, &models.FindFilterType{ PerPage: &pp, @@ -413,7 +561,6 @@ func verifyStudiosGalleryCount(t *testing.T, galleryCountCriterion models.IntCri Studios: &models.HierarchicalMultiCriterionInput{ Value: []string{strconv.Itoa(studio.ID)}, Modifier: models.CriterionModifierIncludes, - Depth: 0, }, }, &models.FindFilterType{ PerPage: &pp, @@ -459,7 +606,7 @@ func TestStudioQueryURL(t *testing.T) { URL: &urlCriterion, } - verifyFn := func(g *models.Studio) { + verifyFn := func(g *models.Studio, r models.Repository) { t.Helper() verifyNullString(t, g.URL, urlCriterion) } @@ -509,7 +656,7 @@ func TestStudioQueryRating(t *testing.T) { verifyStudiosRating(t, ratingCriterion) } -func verifyStudioQuery(t *testing.T, filter models.StudioFilterType, verifyFn func(s *models.Studio)) { +func verifyStudioQuery(t *testing.T, filter models.StudioFilterType, verifyFn func(s *models.Studio, r models.Repository)) { withTxn(func(r models.Repository) error { t.Helper() sqb := r.Studio() @@ -520,7 +667,7 @@ func verifyStudioQuery(t *testing.T, filter models.StudioFilterType, verifyFn fu assert.Greater(t, len(studios), 0) for _, studio := range studios { - verifyFn(studio) + verifyFn(studio, r) } return nil @@ -581,6 +728,189 @@ func queryStudio(t *testing.T, sqb models.StudioReader, studioFilter *models.Stu return studios } +func TestStudioQueryName(t *testing.T) { + const studioIdx = 1 + studioName := getStudioStringValue(studioIdx, "Name") + + nameCriterion := &models.StringCriterionInput{ + Value: studioName, + Modifier: models.CriterionModifierEquals, + } + + studioFilter := models.StudioFilterType{ + Name: nameCriterion, + } + + verifyFn := func(studio *models.Studio, r models.Repository) { + verifyNullString(t, studio.Name, *nameCriterion) + } + + verifyStudioQuery(t, studioFilter, verifyFn) + + nameCriterion.Modifier = models.CriterionModifierNotEquals + verifyStudioQuery(t, studioFilter, verifyFn) + + nameCriterion.Modifier = models.CriterionModifierMatchesRegex + nameCriterion.Value = "studio_.*1_Name" + verifyStudioQuery(t, studioFilter, verifyFn) + + nameCriterion.Modifier = models.CriterionModifierNotMatchesRegex + verifyStudioQuery(t, studioFilter, verifyFn) +} + +func TestStudioQueryAlias(t *testing.T) { + const studioIdx = 1 + studioName := getStudioStringValue(studioIdx, "Alias") + + aliasCriterion := &models.StringCriterionInput{ + Value: studioName, + Modifier: models.CriterionModifierEquals, + } + + studioFilter := models.StudioFilterType{ + Aliases: aliasCriterion, + } + + verifyFn := func(studio *models.Studio, r models.Repository) { + aliases, err := r.Studio().GetAliases(studio.ID) + if err != nil { + t.Errorf("Error querying studios: %s", err.Error()) + } + + var alias string + if len(aliases) > 0 { + alias = aliases[0] + } + + verifyString(t, alias, *aliasCriterion) + } + + verifyStudioQuery(t, studioFilter, verifyFn) + + aliasCriterion.Modifier = models.CriterionModifierNotEquals + verifyStudioQuery(t, studioFilter, verifyFn) + + aliasCriterion.Modifier = models.CriterionModifierMatchesRegex + aliasCriterion.Value = "studio_.*1_Alias" + verifyStudioQuery(t, studioFilter, verifyFn) + + aliasCriterion.Modifier = models.CriterionModifierNotMatchesRegex + verifyStudioQuery(t, studioFilter, verifyFn) +} + +func TestStudioUpdateAlias(t *testing.T) { + if err := withTxn(func(r models.Repository) error { + qb := r.Studio() + + // create studio to test against + const name = "TestStudioUpdateAlias" + created, err := createStudio(qb, name, nil) + if err != nil { + return fmt.Errorf("Error creating studio: %s", err.Error()) + } + + aliases := []string{"alias1", "alias2"} + err = qb.UpdateAliases(created.ID, aliases) + if err != nil { + return fmt.Errorf("Error updating studio aliases: %s", err.Error()) + } + + // ensure aliases set + storedAliases, err := qb.GetAliases(created.ID) + if err != nil { + return fmt.Errorf("Error getting aliases: %s", err.Error()) + } + assert.Equal(t, aliases, storedAliases) + + return nil + }); err != nil { + t.Error(err.Error()) + } +} + +// TestStudioQueryFast does a quick test for major errors, no result verification +func TestStudioQueryFast(t *testing.T) { + + tsString := "test" + tsInt := 1 + + testStringCriterion := models.StringCriterionInput{ + Value: tsString, + Modifier: models.CriterionModifierEquals, + } + testIncludesMultiCriterion := models.MultiCriterionInput{ + Value: []string{tsString}, + Modifier: models.CriterionModifierIncludes, + } + testIntCriterion := models.IntCriterionInput{ + Value: tsInt, + Modifier: models.CriterionModifierEquals, + } + + nameFilter := models.StudioFilterType{ + Name: &testStringCriterion, + } + aliasesFilter := models.StudioFilterType{ + Aliases: &testStringCriterion, + } + stashIDFilter := models.StudioFilterType{ + StashID: &testStringCriterion, + } + urlFilter := models.StudioFilterType{ + URL: &testStringCriterion, + } + ratingFilter := models.StudioFilterType{ + Rating: &testIntCriterion, + } + sceneCountFilter := models.StudioFilterType{ + SceneCount: &testIntCriterion, + } + imageCountFilter := models.StudioFilterType{ + SceneCount: &testIntCriterion, + } + parentsFilter := models.StudioFilterType{ + Parents: &testIncludesMultiCriterion, + } + + filters := []models.StudioFilterType{nameFilter, aliasesFilter, stashIDFilter, urlFilter, ratingFilter, sceneCountFilter, imageCountFilter, parentsFilter} + + missingStrings := []string{"image", "stash_id", "details"} + + for _, m := range missingStrings { + filters = append(filters, models.StudioFilterType{ + IsMissing: &m, + }) + } + + sortbyStrings := []string{"scenes_count", "images_count", "galleries_count", "created_at", "updated_at", "name", "random_26819649", "rating"} + + var findFilters []models.FindFilterType + + for _, sb := range sortbyStrings { + findFilters = append(findFilters, models.FindFilterType{ + Q: &tsString, + Page: &tsInt, + PerPage: &tsInt, + Sort: &sb, + }) + + } + + withTxn(func(r models.Repository) error { + sqb := r.Studio() + for _, f := range filters { + for _, ff := range findFilters { + _, _, err := sqb.Query(&f, &ff) + if err != nil { + t.Errorf("Error querying studio: %s", err.Error()) + } + } + } + + return nil + }) +} + // TODO Create // TODO Update // TODO Destroy diff --git a/pkg/sqlite/tag.go b/pkg/sqlite/tag.go index ca97e5460..0acf748c0 100644 --- a/pkg/sqlite/tag.go +++ b/pkg/sqlite/tag.go @@ -78,7 +78,7 @@ func (qb *tagQueryBuilder) Destroy(id int) error { } if primaryMarkers > 0 { - return errors.New("Cannot delete tag used as a primary tag in scene markers") + return errors.New("cannot delete tag used as a primary tag in scene markers") } return qb.destroyExisting([]int{id}) @@ -196,6 +196,28 @@ func (qb *tagQueryBuilder) FindByNames(names []string, nocase bool) ([]*models.T return qb.queryTags(query, args) } +func (qb *tagQueryBuilder) FindByParentTagID(parentID int) ([]*models.Tag, error) { + query := ` + SELECT tags.* FROM tags + INNER JOIN tags_relations ON tags_relations.child_id = tags.id + WHERE tags_relations.parent_id = ? + ` + query += qb.getDefaultTagSort() + args := []interface{}{parentID} + return qb.queryTags(query, args) +} + +func (qb *tagQueryBuilder) FindByChildTagID(parentID int) ([]*models.Tag, error) { + query := ` + SELECT tags.* FROM tags + INNER JOIN tags_relations ON tags_relations.parent_id = tags.id + WHERE tags_relations.child_id = ? + ` + query += qb.getDefaultTagSort() + args := []interface{}{parentID} + return qb.queryTags(query, args) +} + func (qb *tagQueryBuilder) Count() (int, error) { return qb.runCountQuery(qb.buildCountQuery("SELECT tags.id FROM tags"), nil) } @@ -280,6 +302,10 @@ func (qb *tagQueryBuilder) makeFilter(tagFilter *models.TagFilterType) *filterBu query.handleCriterion(tagGalleryCountCriterionHandler(qb, tagFilter.GalleryCount)) query.handleCriterion(tagPerformerCountCriterionHandler(qb, tagFilter.PerformerCount)) query.handleCriterion(tagMarkerCountCriterionHandler(qb, tagFilter.MarkerCount)) + query.handleCriterion(tagParentsCriterionHandler(qb, tagFilter.Parents)) + query.handleCriterion(tagChildrenCriterionHandler(qb, tagFilter.Children)) + query.handleCriterion(tagParentCountCriterionHandler(qb, tagFilter.ParentCount)) + query.handleCriterion(tagChildCountCriterionHandler(qb, tagFilter.ChildCount)) return query } @@ -411,6 +437,94 @@ func tagMarkerCountCriterionHandler(qb *tagQueryBuilder, markerCount *models.Int } } +func tagParentsCriterionHandler(qb *tagQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if tags != nil && len(tags.Value) > 0 { + var args []interface{} + for _, val := range tags.Value { + args = append(args, val) + } + + depthVal := 0 + if tags.Depth != nil { + depthVal = *tags.Depth + } + + var depthCondition string + if depthVal != -1 { + depthCondition = fmt.Sprintf("WHERE depth < %d", depthVal) + } + + query := `parents AS ( + SELECT parent_id AS root_id, child_id AS item_id, 0 AS depth FROM tags_relations WHERE parent_id IN` + getInBinding(len(tags.Value)) + ` + UNION + SELECT root_id, child_id, depth + 1 FROM tags_relations INNER JOIN parents ON item_id = parent_id ` + depthCondition + ` +)` + + f.addRecursiveWith(query, args...) + + f.addJoin("parents", "", "parents.item_id = tags.id") + + addHierarchicalConditionClauses(f, tags, "parents", "root_id") + } + } +} + +func tagChildrenCriterionHandler(qb *tagQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if tags != nil && len(tags.Value) > 0 { + var args []interface{} + for _, val := range tags.Value { + args = append(args, val) + } + + depthVal := 0 + if tags.Depth != nil { + depthVal = *tags.Depth + } + + var depthCondition string + if depthVal != -1 { + depthCondition = fmt.Sprintf("WHERE depth < %d", depthVal) + } + + query := `children AS ( + SELECT child_id AS root_id, parent_id AS item_id, 0 AS depth FROM tags_relations WHERE child_id IN` + getInBinding(len(tags.Value)) + ` + UNION + SELECT root_id, parent_id, depth + 1 FROM tags_relations INNER JOIN children ON item_id = child_id ` + depthCondition + ` +)` + + f.addRecursiveWith(query, args...) + + f.addJoin("children", "", "children.item_id = tags.id") + + addHierarchicalConditionClauses(f, tags, "children", "root_id") + } + } +} + +func tagParentCountCriterionHandler(qb *tagQueryBuilder, parentCount *models.IntCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if parentCount != nil { + f.addJoin("tags_relations", "parents_count", "parents_count.child_id = tags.id") + clause, args := getIntCriterionWhereClause("count(distinct parents_count.parent_id)", *parentCount) + + f.addHaving(clause, args...) + } + } +} + +func tagChildCountCriterionHandler(qb *tagQueryBuilder, childCount *models.IntCriterionInput) criterionHandlerFunc { + return func(f *filterBuilder) { + if childCount != nil { + f.addJoin("tags_relations", "children_count", "children_count.parent_id = tags.id") + clause, args := getIntCriterionWhereClause("count(distinct children_count.child_id)", *childCount) + + f.addHaving(clause, args...) + } + } +} + func (qb *tagQueryBuilder) getDefaultTagSort() string { return getSort("name", "ASC", "tags") } @@ -429,20 +543,15 @@ func (qb *tagQueryBuilder) getTagSort(query *queryBuilder, findFilter *models.Fi if findFilter.Sort != nil { switch *findFilter.Sort { case "scenes_count": - query.join("scenes_tags", "", "scenes_tags.tag_id = tags.id") - return " ORDER BY COUNT(distinct scenes_tags.scene_id) " + direction + return getCountSort(tagTable, scenesTagsTable, tagIDColumn, direction) case "scene_markers_count": - query.join("scene_markers_tags", "", "scene_markers_tags.tag_id = tags.id") - return " ORDER BY COUNT(distinct scene_markers_tags.scene_marker_id) " + direction + return getCountSort(tagTable, "scene_markers_tags", tagIDColumn, direction) case "images_count": - query.join("images_tags", "", "images_tags.tag_id = tags.id") - return " ORDER BY COUNT(distinct images_tags.image_id) " + direction + return getCountSort(tagTable, imagesTagsTable, tagIDColumn, direction) case "galleries_count": - query.join("galleries_tags", "", "galleries_tags.tag_id = tags.id") - return " ORDER BY COUNT(distinct galleries_tags.gallery_id) " + direction + return getCountSort(tagTable, galleriesTagsTable, tagIDColumn, direction) case "performers_count": - query.join("performers_tags", "", "performers_tags.tag_id = tags.id") - return " ORDER BY COUNT(distinct performers_tags.performer_id) " + direction + return getCountSort(tagTable, performersTagsTable, tagIDColumn, direction) } } @@ -572,3 +681,115 @@ AND NOT EXISTS(SELECT 1 FROM `+table+` o WHERE o.`+idColumn+` = `+table+`.`+idCo return nil } + +func (qb *tagQueryBuilder) UpdateParentTags(tagID int, parentIDs []int) error { + tx := qb.tx + if _, err := tx.Exec("DELETE FROM tags_relations WHERE child_id = ?", tagID); err != nil { + return err + } + + if len(parentIDs) > 0 { + var args []interface{} + var values []string + for _, parentID := range parentIDs { + values = append(values, "(? , ?)") + args = append(args, parentID, tagID) + } + + query := "INSERT INTO tags_relations (parent_id, child_id) VALUES " + strings.Join(values, ", ") + if _, err := tx.Exec(query, args...); err != nil { + return err + } + } + + return nil +} + +func (qb *tagQueryBuilder) UpdateChildTags(tagID int, childIDs []int) error { + tx := qb.tx + if _, err := tx.Exec("DELETE FROM tags_relations WHERE parent_id = ?", tagID); err != nil { + return err + } + + if len(childIDs) > 0 { + var args []interface{} + var values []string + for _, childID := range childIDs { + values = append(values, "(? , ?)") + args = append(args, tagID, childID) + } + + query := "INSERT INTO tags_relations (parent_id, child_id) VALUES " + strings.Join(values, ", ") + if _, err := tx.Exec(query, args...); err != nil { + return err + } + } + + return nil +} + +func (qb *tagQueryBuilder) FindAllAncestors(tagID int, excludeIDs []int) ([]*models.Tag, error) { + inBinding := getInBinding(len(excludeIDs) + 1) + + query := `WITH RECURSIVE +parents AS ( + SELECT t.id AS parent_id, t.id AS child_id FROM tags t WHERE t.id = ? + UNION + SELECT tr.parent_id, tr.child_id FROM tags_relations tr INNER JOIN parents p ON p.parent_id = tr.child_id WHERE tr.parent_id NOT IN` + inBinding + ` +), +children AS ( + SELECT tr.parent_id, tr.child_id FROM tags_relations tr INNER JOIN parents p ON p.parent_id = tr.parent_id WHERE tr.child_id NOT IN` + inBinding + ` + UNION + SELECT tr.parent_id, tr.child_id FROM tags_relations tr INNER JOIN children c ON c.child_id = tr.parent_id WHERE tr.child_id NOT IN` + inBinding + ` +) +SELECT t.* FROM tags t INNER JOIN parents p ON t.id = p.parent_id +UNION +SELECT t.* FROM tags t INNER JOIN children c ON t.id = c.child_id +` + + var ret models.Tags + excludeArgs := []interface{}{tagID} + for _, excludeID := range excludeIDs { + excludeArgs = append(excludeArgs, excludeID) + } + args := []interface{}{tagID} + args = append(args, append(append(excludeArgs, excludeArgs...), excludeArgs...)...) + if err := qb.query(query, args, &ret); err != nil { + return nil, err + } + + return ret, nil +} + +func (qb *tagQueryBuilder) FindAllDescendants(tagID int, excludeIDs []int) ([]*models.Tag, error) { + inBinding := getInBinding(len(excludeIDs) + 1) + + query := `WITH RECURSIVE +children AS ( + SELECT t.id AS parent_id, t.id AS child_id FROM tags t WHERE t.id = ? + UNION + SELECT tr.parent_id, tr.child_id FROM tags_relations tr INNER JOIN children c ON c.child_id = tr.parent_id WHERE tr.child_id NOT IN` + inBinding + ` +), +parents AS ( + SELECT tr.parent_id, tr.child_id FROM tags_relations tr INNER JOIN children c ON c.child_id = tr.child_id WHERE tr.parent_id NOT IN` + inBinding + ` + UNION + SELECT tr.parent_id, tr.child_id FROM tags_relations tr INNER JOIN parents p ON p.parent_id = tr.child_id WHERE tr.parent_id NOT IN` + inBinding + ` +) +SELECT t.* FROM tags t INNER JOIN children c ON t.id = c.child_id +UNION +SELECT t.* FROM tags t INNER JOIN parents p ON t.id = p.parent_id +` + + var ret models.Tags + excludeArgs := []interface{}{tagID} + for _, excludeID := range excludeIDs { + excludeArgs = append(excludeArgs, excludeID) + } + args := []interface{}{tagID} + args = append(args, append(append(excludeArgs, excludeArgs...), excludeArgs...)...) + if err := qb.query(query, args, &ret); err != nil { + return nil, err + } + + return ret, nil +} diff --git a/pkg/sqlite/tag_test.go b/pkg/sqlite/tag_test.go index add3753a7..d91325280 100644 --- a/pkg/sqlite/tag_test.go +++ b/pkg/sqlite/tag_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package sqlite_test @@ -5,6 +6,7 @@ package sqlite_test import ( "database/sql" "fmt" + "strconv" "strings" "testing" @@ -149,6 +151,41 @@ func TestTagFindByNames(t *testing.T) { }) } +func TestTagQuerySort(t *testing.T) { + withTxn(func(r models.Repository) error { + sqb := r.Tag() + + sortBy := "scenes_count" + dir := models.SortDirectionEnumDesc + findFilter := &models.FindFilterType{ + Sort: &sortBy, + Direction: &dir, + } + + tags := queryTags(t, sqb, nil, findFilter) + assert := assert.New(t) + assert.Equal(tagIDs[tagIdxWithScene], tags[0].ID) + + sortBy = "scene_markers_count" + tags = queryTags(t, sqb, nil, findFilter) + assert.Equal(tagIDs[tagIdxWithMarker], tags[0].ID) + + sortBy = "images_count" + tags = queryTags(t, sqb, nil, findFilter) + assert.Equal(tagIDs[tagIdxWithImage], tags[0].ID) + + sortBy = "galleries_count" + tags = queryTags(t, sqb, nil, findFilter) + assert.Equal(tagIDs[tagIdxWithGallery], tags[0].ID) + + sortBy = "performers_count" + tags = queryTags(t, sqb, nil, findFilter) + assert.Equal(tagIDs[tagIdxWithPerformer], tags[0].ID) + + return nil + }) +} + func TestTagQueryName(t *testing.T) { const tagIdx = 1 tagName := getSceneStringValue(tagIdx, "Name") @@ -488,6 +525,198 @@ func verifyTagPerformerCount(t *testing.T, imageCountCriterion models.IntCriteri }) } +func TestTagQueryParentCount(t *testing.T) { + countCriterion := models.IntCriterionInput{ + Value: 1, + Modifier: models.CriterionModifierEquals, + } + + verifyTagParentCount(t, countCriterion) + + countCriterion.Modifier = models.CriterionModifierNotEquals + verifyTagParentCount(t, countCriterion) + + countCriterion.Modifier = models.CriterionModifierLessThan + verifyTagParentCount(t, countCriterion) + + countCriterion.Value = 0 + countCriterion.Modifier = models.CriterionModifierGreaterThan + verifyTagParentCount(t, countCriterion) +} + +func verifyTagParentCount(t *testing.T, sceneCountCriterion models.IntCriterionInput) { + withTxn(func(r models.Repository) error { + qb := r.Tag() + tagFilter := models.TagFilterType{ + ParentCount: &sceneCountCriterion, + } + + tags := queryTags(t, qb, &tagFilter, nil) + + if len(tags) == 0 { + t.Error("Expected at least one tag") + } + + for _, tag := range tags { + verifyInt64(t, sql.NullInt64{ + Int64: int64(getTagParentCount(tag.ID)), + Valid: true, + }, sceneCountCriterion) + } + + return nil + }) +} + +func TestTagQueryChildCount(t *testing.T) { + countCriterion := models.IntCriterionInput{ + Value: 1, + Modifier: models.CriterionModifierEquals, + } + + verifyTagChildCount(t, countCriterion) + + countCriterion.Modifier = models.CriterionModifierNotEquals + verifyTagChildCount(t, countCriterion) + + countCriterion.Modifier = models.CriterionModifierLessThan + verifyTagChildCount(t, countCriterion) + + countCriterion.Value = 0 + countCriterion.Modifier = models.CriterionModifierGreaterThan + verifyTagChildCount(t, countCriterion) +} + +func verifyTagChildCount(t *testing.T, sceneCountCriterion models.IntCriterionInput) { + withTxn(func(r models.Repository) error { + qb := r.Tag() + tagFilter := models.TagFilterType{ + ChildCount: &sceneCountCriterion, + } + + tags := queryTags(t, qb, &tagFilter, nil) + + if len(tags) == 0 { + t.Error("Expected at least one tag") + } + + for _, tag := range tags { + verifyInt64(t, sql.NullInt64{ + Int64: int64(getTagChildCount(tag.ID)), + Valid: true, + }, sceneCountCriterion) + } + + return nil + }) +} + +func TestTagQueryParent(t *testing.T) { + withTxn(func(r models.Repository) error { + sqb := r.Tag() + tagCriterion := models.HierarchicalMultiCriterionInput{ + Value: []string{ + strconv.Itoa(tagIDs[tagIdxWithChildTag]), + }, + Modifier: models.CriterionModifierIncludes, + } + + tagFilter := models.TagFilterType{ + Parents: &tagCriterion, + } + + tags := queryTags(t, sqb, &tagFilter, nil) + + assert.Len(t, tags, 1) + + // ensure id is correct + assert.Equal(t, sceneIDs[tagIdxWithParentTag], tags[0].ID) + + tagCriterion.Modifier = models.CriterionModifierExcludes + + q := getTagStringValue(tagIdxWithParentTag, titleField) + findFilter := models.FindFilterType{ + Q: &q, + } + + tags = queryTags(t, sqb, &tagFilter, &findFilter) + assert.Len(t, tags, 0) + + depth := -1 + + tagCriterion = models.HierarchicalMultiCriterionInput{ + Value: []string{ + strconv.Itoa(tagIDs[tagIdxWithGrandChild]), + }, + Modifier: models.CriterionModifierIncludes, + Depth: &depth, + } + + tags = queryTags(t, sqb, &tagFilter, nil) + assert.Len(t, tags, 2) + + depth = 1 + + tags = queryTags(t, sqb, &tagFilter, nil) + assert.Len(t, tags, 2) + + return nil + }) +} + +func TestTagQueryChild(t *testing.T) { + withTxn(func(r models.Repository) error { + sqb := r.Tag() + tagCriterion := models.HierarchicalMultiCriterionInput{ + Value: []string{ + strconv.Itoa(tagIDs[tagIdxWithParentTag]), + }, + Modifier: models.CriterionModifierIncludes, + } + + tagFilter := models.TagFilterType{ + Children: &tagCriterion, + } + + tags := queryTags(t, sqb, &tagFilter, nil) + + assert.Len(t, tags, 1) + + // ensure id is correct + assert.Equal(t, sceneIDs[tagIdxWithChildTag], tags[0].ID) + + tagCriterion.Modifier = models.CriterionModifierExcludes + + q := getTagStringValue(tagIdxWithChildTag, titleField) + findFilter := models.FindFilterType{ + Q: &q, + } + + tags = queryTags(t, sqb, &tagFilter, &findFilter) + assert.Len(t, tags, 0) + + depth := -1 + + tagCriterion = models.HierarchicalMultiCriterionInput{ + Value: []string{ + strconv.Itoa(tagIDs[tagIdxWithGrandParent]), + }, + Modifier: models.CriterionModifierIncludes, + Depth: &depth, + } + + tags = queryTags(t, sqb, &tagFilter, nil) + assert.Len(t, tags, 2) + + depth = 1 + + tags = queryTags(t, sqb, &tagFilter, nil) + assert.Len(t, tags, 2) + + return nil + }) +} + func TestTagUpdateTagImage(t *testing.T) { if err := withTxn(func(r models.Repository) error { qb := r.Tag() diff --git a/pkg/static/embed.go b/pkg/static/embed.go new file mode 100644 index 000000000..a0563e6ac --- /dev/null +++ b/pkg/static/embed.go @@ -0,0 +1,9 @@ +package static + +import "embed" + +//go:embed performer +var Performer embed.FS + +//go:embed performer_male +var PerformerMale embed.FS diff --git a/static/performer/NoName01.png b/pkg/static/performer/NoName01.png similarity index 100% rename from static/performer/NoName01.png rename to pkg/static/performer/NoName01.png diff --git a/static/performer/NoName02.png b/pkg/static/performer/NoName02.png similarity index 100% rename from static/performer/NoName02.png rename to pkg/static/performer/NoName02.png diff --git a/static/performer/NoName03.png b/pkg/static/performer/NoName03.png similarity index 100% rename from static/performer/NoName03.png rename to pkg/static/performer/NoName03.png diff --git a/static/performer/NoName04.png b/pkg/static/performer/NoName04.png similarity index 100% rename from static/performer/NoName04.png rename to pkg/static/performer/NoName04.png diff --git a/static/performer/NoName05.png b/pkg/static/performer/NoName05.png similarity index 100% rename from static/performer/NoName05.png rename to pkg/static/performer/NoName05.png diff --git a/static/performer/NoName06.png b/pkg/static/performer/NoName06.png similarity index 100% rename from static/performer/NoName06.png rename to pkg/static/performer/NoName06.png diff --git a/static/performer/NoName07.png b/pkg/static/performer/NoName07.png similarity index 100% rename from static/performer/NoName07.png rename to pkg/static/performer/NoName07.png diff --git a/static/performer/NoName08.png b/pkg/static/performer/NoName08.png similarity index 100% rename from static/performer/NoName08.png rename to pkg/static/performer/NoName08.png diff --git a/static/performer/NoName09.png b/pkg/static/performer/NoName09.png similarity index 100% rename from static/performer/NoName09.png rename to pkg/static/performer/NoName09.png diff --git a/static/performer/NoName10.png b/pkg/static/performer/NoName10.png similarity index 100% rename from static/performer/NoName10.png rename to pkg/static/performer/NoName10.png diff --git a/static/performer/NoName11.png b/pkg/static/performer/NoName11.png similarity index 100% rename from static/performer/NoName11.png rename to pkg/static/performer/NoName11.png diff --git a/static/performer/NoName12.png b/pkg/static/performer/NoName12.png similarity index 100% rename from static/performer/NoName12.png rename to pkg/static/performer/NoName12.png diff --git a/static/performer/NoName13.png b/pkg/static/performer/NoName13.png similarity index 100% rename from static/performer/NoName13.png rename to pkg/static/performer/NoName13.png diff --git a/static/performer/NoName14.png b/pkg/static/performer/NoName14.png similarity index 100% rename from static/performer/NoName14.png rename to pkg/static/performer/NoName14.png diff --git a/static/performer/NoName15.png b/pkg/static/performer/NoName15.png similarity index 100% rename from static/performer/NoName15.png rename to pkg/static/performer/NoName15.png diff --git a/static/performer/NoName16.png b/pkg/static/performer/NoName16.png similarity index 100% rename from static/performer/NoName16.png rename to pkg/static/performer/NoName16.png diff --git a/static/performer/NoName17.png b/pkg/static/performer/NoName17.png similarity index 100% rename from static/performer/NoName17.png rename to pkg/static/performer/NoName17.png diff --git a/static/performer/NoName18.png b/pkg/static/performer/NoName18.png similarity index 100% rename from static/performer/NoName18.png rename to pkg/static/performer/NoName18.png diff --git a/static/performer/NoName19.png b/pkg/static/performer/NoName19.png similarity index 100% rename from static/performer/NoName19.png rename to pkg/static/performer/NoName19.png diff --git a/static/performer/NoName20.png b/pkg/static/performer/NoName20.png similarity index 100% rename from static/performer/NoName20.png rename to pkg/static/performer/NoName20.png diff --git a/static/performer/NoName21.png b/pkg/static/performer/NoName21.png similarity index 100% rename from static/performer/NoName21.png rename to pkg/static/performer/NoName21.png diff --git a/static/performer/NoName22.png b/pkg/static/performer/NoName22.png similarity index 100% rename from static/performer/NoName22.png rename to pkg/static/performer/NoName22.png diff --git a/static/performer/NoName23.png b/pkg/static/performer/NoName23.png similarity index 100% rename from static/performer/NoName23.png rename to pkg/static/performer/NoName23.png diff --git a/static/performer/NoName24.png b/pkg/static/performer/NoName24.png similarity index 100% rename from static/performer/NoName24.png rename to pkg/static/performer/NoName24.png diff --git a/static/performer/NoName25.png b/pkg/static/performer/NoName25.png similarity index 100% rename from static/performer/NoName25.png rename to pkg/static/performer/NoName25.png diff --git a/static/performer/NoName26.png b/pkg/static/performer/NoName26.png similarity index 100% rename from static/performer/NoName26.png rename to pkg/static/performer/NoName26.png diff --git a/static/performer/NoName27.png b/pkg/static/performer/NoName27.png similarity index 100% rename from static/performer/NoName27.png rename to pkg/static/performer/NoName27.png diff --git a/static/performer/NoName28.png b/pkg/static/performer/NoName28.png similarity index 100% rename from static/performer/NoName28.png rename to pkg/static/performer/NoName28.png diff --git a/static/performer/NoName29.png b/pkg/static/performer/NoName29.png similarity index 100% rename from static/performer/NoName29.png rename to pkg/static/performer/NoName29.png diff --git a/static/performer/NoName30.png b/pkg/static/performer/NoName30.png similarity index 100% rename from static/performer/NoName30.png rename to pkg/static/performer/NoName30.png diff --git a/static/performer/NoName31.png b/pkg/static/performer/NoName31.png similarity index 100% rename from static/performer/NoName31.png rename to pkg/static/performer/NoName31.png diff --git a/static/performer/NoName32.png b/pkg/static/performer/NoName32.png similarity index 100% rename from static/performer/NoName32.png rename to pkg/static/performer/NoName32.png diff --git a/static/performer/NoName33.png b/pkg/static/performer/NoName33.png similarity index 100% rename from static/performer/NoName33.png rename to pkg/static/performer/NoName33.png diff --git a/static/performer/NoName34.png b/pkg/static/performer/NoName34.png similarity index 100% rename from static/performer/NoName34.png rename to pkg/static/performer/NoName34.png diff --git a/static/performer/NoName35.png b/pkg/static/performer/NoName35.png similarity index 100% rename from static/performer/NoName35.png rename to pkg/static/performer/NoName35.png diff --git a/static/performer/NoName36.png b/pkg/static/performer/NoName36.png similarity index 100% rename from static/performer/NoName36.png rename to pkg/static/performer/NoName36.png diff --git a/static/performer/NoName37.png b/pkg/static/performer/NoName37.png similarity index 100% rename from static/performer/NoName37.png rename to pkg/static/performer/NoName37.png diff --git a/static/performer/NoName38.png b/pkg/static/performer/NoName38.png similarity index 100% rename from static/performer/NoName38.png rename to pkg/static/performer/NoName38.png diff --git a/static/performer/NoName39.png b/pkg/static/performer/NoName39.png similarity index 100% rename from static/performer/NoName39.png rename to pkg/static/performer/NoName39.png diff --git a/static/performer/NoName40.png b/pkg/static/performer/NoName40.png similarity index 100% rename from static/performer/NoName40.png rename to pkg/static/performer/NoName40.png diff --git a/static/performer_male/noname_male_01.jpg b/pkg/static/performer_male/noname_male_01.jpg similarity index 100% rename from static/performer_male/noname_male_01.jpg rename to pkg/static/performer_male/noname_male_01.jpg diff --git a/static/performer_male/noname_male_02.jpg b/pkg/static/performer_male/noname_male_02.jpg similarity index 100% rename from static/performer_male/noname_male_02.jpg rename to pkg/static/performer_male/noname_male_02.jpg diff --git a/pkg/studio/export.go b/pkg/studio/export.go index dc71fd915..46b92a07d 100644 --- a/pkg/studio/export.go +++ b/pkg/studio/export.go @@ -42,6 +42,13 @@ func ToJSON(reader models.StudioReader, studio *models.Studio) (*jsonschema.Stud newStudioJSON.Rating = int(studio.Rating.Int64) } + aliases, err := reader.GetAliases(studio.ID) + if err != nil { + return nil, fmt.Errorf("error getting studio aliases: %s", err.Error()) + } + + newStudioJSON.Aliases = aliases + image, err := reader.GetImage(studio.ID) if err != nil { return nil, fmt.Errorf("error getting studio image: %s", err.Error()) diff --git a/pkg/studio/export_test.go b/pkg/studio/export_test.go index 516c3714e..e251ad52c 100644 --- a/pkg/studio/export_test.go +++ b/pkg/studio/export_test.go @@ -18,6 +18,7 @@ const ( errImageID = 3 missingParentStudioID = 4 errStudioID = 5 + errAliasID = 6 parentStudioID = 10 missingStudioID = 11 @@ -40,8 +41,10 @@ var imageBytes = []byte("imageBytes") const image = "aW1hZ2VCeXRlcw==" -var createTime time.Time = time.Date(2001, 01, 01, 0, 0, 0, 0, time.Local) -var updateTime time.Time = time.Date(2002, 01, 01, 0, 0, 0, 0, time.Local) +var ( + createTime = time.Date(2001, 01, 01, 0, 0, 0, 0, time.Local) + updateTime = time.Date(2002, 01, 01, 0, 0, 0, 0, time.Local) +) func createFullStudio(id int, parentID int) models.Studio { ret := models.Studio{ @@ -77,7 +80,7 @@ func createEmptyStudio(id int) models.Studio { } } -func createFullJSONStudio(parentStudio, image string) *jsonschema.Studio { +func createFullJSONStudio(parentStudio, image string, aliases []string) *jsonschema.Studio { return &jsonschema.Studio{ Name: studioName, URL: url, @@ -91,6 +94,7 @@ func createFullJSONStudio(parentStudio, image string) *jsonschema.Studio { ParentStudio: parentStudio, Image: image, Rating: rating, + Aliases: aliases, } } @@ -115,31 +119,36 @@ var scenarios []testScenario func initTestTable() { scenarios = []testScenario{ - testScenario{ + { createFullStudio(studioID, parentStudioID), - createFullJSONStudio(parentStudioName, image), + createFullJSONStudio(parentStudioName, image, []string{"alias"}), false, }, - testScenario{ + { createEmptyStudio(noImageID), createEmptyJSONStudio(), false, }, - testScenario{ + { createFullStudio(errImageID, parentStudioID), nil, true, }, - testScenario{ + { createFullStudio(missingParentStudioID, missingStudioID), - createFullJSONStudio("", image), + createFullJSONStudio("", image, nil), false, }, - testScenario{ + { createFullStudio(errStudioID, errParentStudioID), nil, true, }, + { + createFullStudio(errAliasID, parentStudioID), + nil, + true, + }, } } @@ -155,6 +164,7 @@ func TestToJSON(t *testing.T) { mockStudioReader.On("GetImage", errImageID).Return(nil, imageErr).Once() mockStudioReader.On("GetImage", missingParentStudioID).Return(imageBytes, nil).Maybe() mockStudioReader.On("GetImage", errStudioID).Return(imageBytes, nil).Maybe() + mockStudioReader.On("GetImage", errAliasID).Return(imageBytes, nil).Maybe() parentStudioErr := errors.New("error getting parent studio") @@ -162,6 +172,14 @@ func TestToJSON(t *testing.T) { mockStudioReader.On("Find", missingStudioID).Return(nil, nil) mockStudioReader.On("Find", errParentStudioID).Return(nil, parentStudioErr) + aliasErr := errors.New("error getting aliases") + + mockStudioReader.On("GetAliases", studioID).Return([]string{"alias"}, nil).Once() + mockStudioReader.On("GetAliases", noImageID).Return(nil, nil).Once() + mockStudioReader.On("GetAliases", errImageID).Return(nil, nil).Once() + mockStudioReader.On("GetAliases", missingParentStudioID).Return(nil, nil).Once() + mockStudioReader.On("GetAliases", errAliasID).Return(nil, aliasErr).Once() + for i, s := range scenarios { studio := s.input json, err := ToJSON(mockStudioReader, &studio) diff --git a/pkg/studio/import.go b/pkg/studio/import.go index f509c0626..a3a35023d 100644 --- a/pkg/studio/import.go +++ b/pkg/studio/import.go @@ -101,6 +101,10 @@ func (i *Importer) PostImport(id int) error { } } + if err := i.ReaderWriter.UpdateAliases(id, i.Input.Aliases); err != nil { + return fmt.Errorf("error setting tag aliases: %s", err.Error()) + } + return nil } diff --git a/pkg/studio/import_test.go b/pkg/studio/import_test.go index 29a0d8813..78c788fd0 100644 --- a/pkg/studio/import_test.go +++ b/pkg/studio/import_test.go @@ -53,7 +53,7 @@ func TestImporterPreImport(t *testing.T) { assert.Nil(t, err) - i.Input = *createFullJSONStudio(studioName, image) + i.Input = *createFullJSONStudio(studioName, image, []string{"alias"}) i.Input.ParentStudio = "" err = i.PreImport() @@ -151,13 +151,22 @@ func TestImporterPostImport(t *testing.T) { i := Importer{ ReaderWriter: readerWriter, - imageData: imageBytes, + Input: jsonschema.Studio{ + Aliases: []string{"alias"}, + }, + imageData: imageBytes, } updateStudioImageErr := errors.New("UpdateImage error") + updateTagAliasErr := errors.New("UpdateAlias error") readerWriter.On("UpdateImage", studioID, imageBytes).Return(nil).Once() readerWriter.On("UpdateImage", errImageID, imageBytes).Return(updateStudioImageErr).Once() + readerWriter.On("UpdateImage", errAliasID, imageBytes).Return(nil).Once() + + readerWriter.On("UpdateAliases", studioID, i.Input.Aliases).Return(nil).Once() + readerWriter.On("UpdateAliases", errImageID, i.Input.Aliases).Return(nil).Maybe() + readerWriter.On("UpdateAliases", errAliasID, i.Input.Aliases).Return(updateTagAliasErr).Once() err := i.PostImport(studioID) assert.Nil(t, err) @@ -165,6 +174,9 @@ func TestImporterPostImport(t *testing.T) { err = i.PostImport(errImageID) assert.NotNil(t, err) + err = i.PostImport(errAliasID) + assert.NotNil(t, err) + readerWriter.AssertExpectations(t) } diff --git a/pkg/studio/query.go b/pkg/studio/query.go new file mode 100644 index 000000000..5b2f68896 --- /dev/null +++ b/pkg/studio/query.go @@ -0,0 +1,51 @@ +package studio + +import "github.com/stashapp/stash/pkg/models" + +func ByName(qb models.StudioReader, name string) (*models.Studio, error) { + f := &models.StudioFilterType{ + Name: &models.StringCriterionInput{ + Value: name, + Modifier: models.CriterionModifierEquals, + }, + } + + pp := 1 + ret, count, err := qb.Query(f, &models.FindFilterType{ + PerPage: &pp, + }) + + if err != nil { + return nil, err + } + + if count > 0 { + return ret[0], nil + } + + return nil, nil +} + +func ByAlias(qb models.StudioReader, alias string) (*models.Studio, error) { + f := &models.StudioFilterType{ + Aliases: &models.StringCriterionInput{ + Value: alias, + Modifier: models.CriterionModifierEquals, + }, + } + + pp := 1 + ret, count, err := qb.Query(f, &models.FindFilterType{ + PerPage: &pp, + }) + + if err != nil { + return nil, err + } + + if count > 0 { + return ret[0], nil + } + + return nil, nil +} diff --git a/pkg/studio/update.go b/pkg/studio/update.go new file mode 100644 index 000000000..35a655a73 --- /dev/null +++ b/pkg/studio/update.go @@ -0,0 +1,65 @@ +package studio + +import ( + "fmt" + + "github.com/stashapp/stash/pkg/models" +) + +type NameExistsError struct { + Name string +} + +func (e *NameExistsError) Error() string { + return fmt.Sprintf("studio with name '%s' already exists", e.Name) +} + +type NameUsedByAliasError struct { + Name string + OtherStudio string +} + +func (e *NameUsedByAliasError) Error() string { + return fmt.Sprintf("name '%s' is used as alias for '%s'", e.Name, e.OtherStudio) +} + +// EnsureStudioNameUnique returns an error if the studio name provided +// is used as a name or alias of another existing tag. +func EnsureStudioNameUnique(id int, name string, qb models.StudioReader) error { + // ensure name is unique + sameNameStudio, err := ByName(qb, name) + if err != nil { + return err + } + + if sameNameStudio != nil && id != sameNameStudio.ID { + return &NameExistsError{ + Name: name, + } + } + + // query by alias + sameNameStudio, err = ByAlias(qb, name) + if err != nil { + return err + } + + if sameNameStudio != nil && id != sameNameStudio.ID { + return &NameUsedByAliasError{ + Name: name, + OtherStudio: sameNameStudio.Name.String, + } + } + + return nil +} + +func EnsureAliasesUnique(id int, aliases []string, qb models.StudioReader) error { + for _, a := range aliases { + if err := EnsureStudioNameUnique(id, a, qb); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/tag/export.go b/pkg/tag/export.go index ba9d6da82..54c64990e 100644 --- a/pkg/tag/export.go +++ b/pkg/tag/export.go @@ -32,6 +32,13 @@ func ToJSON(reader models.TagReader, tag *models.Tag) (*jsonschema.Tag, error) { newTagJSON.Image = utils.GetBase64StringFromData(image) } + parents, err := reader.FindByChildTagID(tag.ID) + if err != nil { + return nil, fmt.Errorf("error getting parents: %s", err.Error()) + } + + newTagJSON.Parents = GetNames(parents) + return &newTagJSON, nil } diff --git a/pkg/tag/export_test.go b/pkg/tag/export_test.go index 16e85292d..2057ccad3 100644 --- a/pkg/tag/export_test.go +++ b/pkg/tag/export_test.go @@ -13,16 +13,20 @@ import ( ) const ( - tagID = 1 - noImageID = 2 - errImageID = 3 - errAliasID = 4 + tagID = 1 + noImageID = 2 + errImageID = 3 + errAliasID = 4 + withParentsID = 5 + errParentsID = 6 ) const tagName = "testTag" -var createTime time.Time = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) -var updateTime time.Time = time.Date(2002, 01, 01, 0, 0, 0, 0, time.UTC) +var ( + createTime = time.Date(2001, 01, 01, 0, 0, 0, 0, time.UTC) + updateTime = time.Date(2002, 01, 01, 0, 0, 0, 0, time.UTC) +) func createTag(id int) models.Tag { return models.Tag{ @@ -37,7 +41,7 @@ func createTag(id int) models.Tag { } } -func createJSONTag(aliases []string, image string) *jsonschema.Tag { +func createJSONTag(aliases []string, image string, parents []string) *jsonschema.Tag { return &jsonschema.Tag{ Name: tagName, Aliases: aliases, @@ -47,7 +51,8 @@ func createJSONTag(aliases []string, image string) *jsonschema.Tag { UpdatedAt: models.JSONTime{ Time: updateTime, }, - Image: image, + Image: image, + Parents: parents, } } @@ -63,12 +68,12 @@ func initTestTable() { scenarios = []testScenario{ { createTag(tagID), - createJSONTag([]string{"alias"}, "PHN2ZwogICB4bWxuczpkYz0iaHR0cDovL3B1cmwub3JnL2RjL2VsZW1lbnRzLzEuMS8iCiAgIHhtbG5zOmNjPSJodHRwOi8vY3JlYXRpdmVjb21tb25zLm9yZy9ucyMiCiAgIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyIKICAgeG1sbnM6c3ZnPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIgogICB4bWxuczpzb2RpcG9kaT0iaHR0cDovL3NvZGlwb2RpLnNvdXJjZWZvcmdlLm5ldC9EVEQvc29kaXBvZGktMC5kdGQiCiAgIHhtbG5zOmlua3NjYXBlPSJodHRwOi8vd3d3Lmlua3NjYXBlLm9yZy9uYW1lc3BhY2VzL2lua3NjYXBlIgogICB3aWR0aD0iMjAwIgogICBoZWlnaHQ9IjIwMCIKICAgaWQ9InN2ZzIiCiAgIHZlcnNpb249IjEuMSIKICAgaW5rc2NhcGU6dmVyc2lvbj0iMC40OC40IHI5OTM5IgogICBzb2RpcG9kaTpkb2NuYW1lPSJ0YWcuc3ZnIj4KICA8ZGVmcwogICAgIGlkPSJkZWZzNCIgLz4KICA8c29kaXBvZGk6bmFtZWR2aWV3CiAgICAgaWQ9ImJhc2UiCiAgICAgcGFnZWNvbG9yPSIjMDAwMDAwIgogICAgIGJvcmRlcmNvbG9yPSIjNjY2NjY2IgogICAgIGJvcmRlcm9wYWNpdHk9IjEuMCIKICAgICBpbmtzY2FwZTpwYWdlb3BhY2l0eT0iMSIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnpvb209IjEiCiAgICAgaW5rc2NhcGU6Y3g9IjE4MS43Nzc3MSIKICAgICBpbmtzY2FwZTpjeT0iMjc5LjcyMzc2IgogICAgIGlua3NjYXBlOmRvY3VtZW50LXVuaXRzPSJweCIKICAgICBpbmtzY2FwZTpjdXJyZW50LWxheWVyPSJsYXllcjEiCiAgICAgc2hvd2dyaWQ9ImZhbHNlIgogICAgIGZpdC1tYXJnaW4tdG9wPSIwIgogICAgIGZpdC1tYXJnaW4tbGVmdD0iMCIKICAgICBmaXQtbWFyZ2luLXJpZ2h0PSIwIgogICAgIGZpdC1tYXJnaW4tYm90dG9tPSIwIgogICAgIGlua3NjYXBlOndpbmRvdy13aWR0aD0iMTkyMCIKICAgICBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDE3IgogICAgIGlua3NjYXBlOndpbmRvdy14PSItOCIKICAgICBpbmtzY2FwZTp3aW5kb3cteT0iLTgiCiAgICAgaW5rc2NhcGU6d2luZG93LW1heGltaXplZD0iMSIgLz4KICA8bWV0YWRhdGEKICAgICBpZD0ibWV0YWRhdGE3Ij4KICAgIDxyZGY6UkRGPgogICAgICA8Y2M6V29yawogICAgICAgICByZGY6YWJvdXQ9IiI+CiAgICAgICAgPGRjOmZvcm1hdD5pbWFnZS9zdmcreG1sPC9kYzpmb3JtYXQ+CiAgICAgICAgPGRjOnR5cGUKICAgICAgICAgICByZGY6cmVzb3VyY2U9Imh0dHA6Ly9wdXJsLm9yZy9kYy9kY21pdHlwZS9TdGlsbEltYWdlIiAvPgogICAgICAgIDxkYzp0aXRsZT48L2RjOnRpdGxlPgogICAgICA8L2NjOldvcms+CiAgICA8L3JkZjpSREY+CiAgPC9tZXRhZGF0YT4KICA8ZwogICAgIGlua3NjYXBlOmxhYmVsPSJMYXllciAxIgogICAgIGlua3NjYXBlOmdyb3VwbW9kZT0ibGF5ZXIiCiAgICAgaWQ9ImxheWVyMSIKICAgICB0cmFuc2Zvcm09InRyYW5zbGF0ZSgtMTU3Ljg0MzU4LC01MjQuNjk1MjIpIj4KICAgIDxwYXRoCiAgICAgICBpZD0icGF0aDI5ODciCiAgICAgICBkPSJtIDIyOS45NDMxNCw2NjkuMjY1NDkgLTM2LjA4NDY2LC0zNi4wODQ2NiBjIC00LjY4NjUzLC00LjY4NjUzIC00LjY4NjUzLC0xMi4yODQ2OCAwLC0xNi45NzEyMSBsIDM2LjA4NDY2LC0zNi4wODQ2NyBhIDEyLjAwMDQ1MywxMi4wMDA0NTMgMCAwIDEgOC40ODU2LC0zLjUxNDggbCA3NC45MTQ0MywwIGMgNi42Mjc2MSwwIDEyLjAwMDQxLDUuMzcyOCAxMi4wMDA0MSwxMi4wMDA0MSBsIDAsNzIuMTY5MzMgYyAwLDYuNjI3NjEgLTUuMzcyOCwxMi4wMDA0MSAtMTIuMDAwNDEsMTIuMDAwNDEgbCAtNzQuOTE0NDMsMCBhIDEyLjAwMDQ1MywxMi4wMDA0NTMgMCAwIDEgLTguNDg1NiwtMy41MTQ4MSB6IG0gLTEzLjQ1NjM5LC01My4wNTU4NyBjIC00LjY4NjUzLDQuNjg2NTMgLTQuNjg2NTMsMTIuMjg0NjggMCwxNi45NzEyMSA0LjY4NjUyLDQuNjg2NTIgMTIuMjg0NjcsNC42ODY1MiAxNi45NzEyLDAgNC42ODY1MywtNC42ODY1MyA0LjY4NjUzLC0xMi4yODQ2OCAwLC0xNi45NzEyMSAtNC42ODY1MywtNC42ODY1MiAtMTIuMjg0NjgsLTQuNjg2NTIgLTE2Ljk3MTIsMCB6IgogICAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIKICAgICAgIHN0eWxlPSJmaWxsOiNmZmZmZmY7ZmlsbC1vcGFjaXR5OjEiIC8+CiAgPC9nPgo8L3N2Zz4="), + createJSONTag([]string{"alias"}, image, nil), false, }, { createTag(noImageID), - createJSONTag(nil, ""), + createJSONTag(nil, "", nil), false, }, { @@ -81,6 +86,16 @@ func initTestTable() { nil, true, }, + { + createTag(withParentsID), + createJSONTag(nil, image, []string{"parent"}), + false, + }, + { + createTag(errParentsID), + nil, + true, + }, } } @@ -91,15 +106,25 @@ func TestToJSON(t *testing.T) { imageErr := errors.New("error getting image") aliasErr := errors.New("error getting aliases") + parentsErr := errors.New("error getting parents") mockTagReader.On("GetAliases", tagID).Return([]string{"alias"}, nil).Once() mockTagReader.On("GetAliases", noImageID).Return(nil, nil).Once() mockTagReader.On("GetAliases", errImageID).Return(nil, nil).Once() mockTagReader.On("GetAliases", errAliasID).Return(nil, aliasErr).Once() + mockTagReader.On("GetAliases", withParentsID).Return(nil, nil).Once() + mockTagReader.On("GetAliases", errParentsID).Return(nil, nil).Once() - mockTagReader.On("GetImage", tagID).Return(models.DefaultTagImage, nil).Once() + mockTagReader.On("GetImage", tagID).Return(imageBytes, nil).Once() mockTagReader.On("GetImage", noImageID).Return(nil, nil).Once() mockTagReader.On("GetImage", errImageID).Return(nil, imageErr).Once() + mockTagReader.On("GetImage", withParentsID).Return(imageBytes, nil).Once() + mockTagReader.On("GetImage", errParentsID).Return(nil, nil).Once() + + mockTagReader.On("FindByChildTagID", tagID).Return(nil, nil).Once() + mockTagReader.On("FindByChildTagID", noImageID).Return(nil, nil).Once() + mockTagReader.On("FindByChildTagID", withParentsID).Return([]*models.Tag{{Name: "parent"}}, nil).Once() + mockTagReader.On("FindByChildTagID", errParentsID).Return(nil, parentsErr).Once() for i, s := range scenarios { tag := s.tag diff --git a/pkg/tag/import.go b/pkg/tag/import.go index 54de4bd0e..8fe0410d2 100644 --- a/pkg/tag/import.go +++ b/pkg/tag/import.go @@ -8,9 +8,22 @@ import ( "github.com/stashapp/stash/pkg/utils" ) +type ParentTagNotExistError struct { + missingParent string +} + +func (e ParentTagNotExistError) Error() string { + return fmt.Sprintf("parent tag <%s> does not exist", e.missingParent) +} + +func (e ParentTagNotExistError) MissingParent() string { + return e.missingParent +} + type Importer struct { - ReaderWriter models.TagReaderWriter - Input jsonschema.Tag + ReaderWriter models.TagReaderWriter + Input jsonschema.Tag + MissingRefBehaviour models.ImportMissingRefEnum tag models.Tag imageData []byte @@ -45,6 +58,15 @@ func (i *Importer) PostImport(id int) error { return fmt.Errorf("error setting tag aliases: %s", err.Error()) } + parents, err := i.getParents() + if err != nil { + return err + } + + if err := i.ReaderWriter.UpdateParentTags(id, parents); err != nil { + return fmt.Errorf("error setting parents: %s", err.Error()) + } + return nil } @@ -87,3 +109,46 @@ func (i *Importer) Update(id int) error { return nil } + +func (i *Importer) getParents() ([]int, error) { + var parents []int + for _, parent := range i.Input.Parents { + tag, err := i.ReaderWriter.FindByName(parent, false) + if err != nil { + return nil, fmt.Errorf("error finding parent by name: %s", err.Error()) + } + + if tag == nil { + if i.MissingRefBehaviour == models.ImportMissingRefEnumFail { + return nil, ParentTagNotExistError{missingParent: parent} + } + + if i.MissingRefBehaviour == models.ImportMissingRefEnumIgnore { + continue + } + + if i.MissingRefBehaviour == models.ImportMissingRefEnumCreate { + parentID, err := i.createParent(parent) + if err != nil { + return nil, err + } + parents = append(parents, parentID) + } + } else { + parents = append(parents, tag.ID) + } + } + + return parents, nil +} + +func (i *Importer) createParent(name string) (int, error) { + newTag := *models.NewTag(name) + + created, err := i.ReaderWriter.Create(newTag) + if err != nil { + return 0, err + } + + return created.ID, nil +} diff --git a/pkg/tag/import_test.go b/pkg/tag/import_test.go index ea29e47c3..c2f29d8e5 100644 --- a/pkg/tag/import_test.go +++ b/pkg/tag/import_test.go @@ -8,6 +8,7 @@ import ( "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) const image = "aW1hZ2VCeXRlcw==" @@ -64,13 +65,25 @@ func TestImporterPostImport(t *testing.T) { updateTagImageErr := errors.New("UpdateImage error") updateTagAliasErr := errors.New("UpdateAlias error") + updateTagParentsErr := errors.New("UpdateParentTags error") readerWriter.On("UpdateAliases", tagID, i.Input.Aliases).Return(nil).Once() readerWriter.On("UpdateAliases", errAliasID, i.Input.Aliases).Return(updateTagAliasErr).Once() + readerWriter.On("UpdateAliases", withParentsID, i.Input.Aliases).Return(nil).Once() + readerWriter.On("UpdateAliases", errParentsID, i.Input.Aliases).Return(nil).Once() readerWriter.On("UpdateImage", tagID, imageBytes).Return(nil).Once() readerWriter.On("UpdateImage", errAliasID, imageBytes).Return(nil).Once() readerWriter.On("UpdateImage", errImageID, imageBytes).Return(updateTagImageErr).Once() + readerWriter.On("UpdateImage", withParentsID, imageBytes).Return(nil).Once() + readerWriter.On("UpdateImage", errParentsID, imageBytes).Return(nil).Once() + + var parentTags []int + readerWriter.On("UpdateParentTags", tagID, parentTags).Return(nil).Once() + readerWriter.On("UpdateParentTags", withParentsID, []int{100}).Return(nil).Once() + readerWriter.On("UpdateParentTags", errParentsID, []int{100}).Return(updateTagParentsErr).Once() + + readerWriter.On("FindByName", "Parent", false).Return(&models.Tag{ID: 100}, nil) err := i.PostImport(tagID) assert.Nil(t, err) @@ -81,6 +94,106 @@ func TestImporterPostImport(t *testing.T) { err = i.PostImport(errAliasID) assert.NotNil(t, err) + i.Input.Parents = []string{"Parent"} + err = i.PostImport(withParentsID) + assert.Nil(t, err) + + err = i.PostImport(errParentsID) + assert.NotNil(t, err) + + readerWriter.AssertExpectations(t) +} + +func TestImporterPostImportParentMissing(t *testing.T) { + readerWriter := &mocks.TagReaderWriter{} + + i := Importer{ + ReaderWriter: readerWriter, + Input: jsonschema.Tag{}, + imageData: imageBytes, + } + + createID := 1 + createErrorID := 2 + createFindErrorID := 3 + createFoundID := 4 + failID := 5 + failFindErrorID := 6 + failFoundID := 7 + ignoreID := 8 + ignoreFindErrorID := 9 + ignoreFoundID := 10 + + findError := errors.New("failed finding parent") + + var emptyParents []int + + readerWriter.On("UpdateImage", mock.Anything, mock.Anything).Return(nil) + readerWriter.On("UpdateAliases", mock.Anything, mock.Anything).Return(nil) + + readerWriter.On("FindByName", "Create", false).Return(nil, nil).Once() + readerWriter.On("FindByName", "CreateError", false).Return(nil, nil).Once() + readerWriter.On("FindByName", "CreateFindError", false).Return(nil, findError).Once() + readerWriter.On("FindByName", "CreateFound", false).Return(&models.Tag{ID: 101}, nil).Once() + readerWriter.On("FindByName", "Fail", false).Return(nil, nil).Once() + readerWriter.On("FindByName", "FailFindError", false).Return(nil, findError) + readerWriter.On("FindByName", "FailFound", false).Return(&models.Tag{ID: 102}, nil).Once() + readerWriter.On("FindByName", "Ignore", false).Return(nil, nil).Once() + readerWriter.On("FindByName", "IgnoreFindError", false).Return(nil, findError) + readerWriter.On("FindByName", "IgnoreFound", false).Return(&models.Tag{ID: 103}, nil).Once() + + readerWriter.On("UpdateParentTags", createID, []int{100}).Return(nil).Once() + readerWriter.On("UpdateParentTags", createFoundID, []int{101}).Return(nil).Once() + readerWriter.On("UpdateParentTags", failFoundID, []int{102}).Return(nil).Once() + readerWriter.On("UpdateParentTags", ignoreID, emptyParents).Return(nil).Once() + readerWriter.On("UpdateParentTags", ignoreFoundID, []int{103}).Return(nil).Once() + + readerWriter.On("Create", mock.MatchedBy(func(t models.Tag) bool { return t.Name == "Create" })).Return(&models.Tag{ID: 100}, nil).Once() + readerWriter.On("Create", mock.MatchedBy(func(t models.Tag) bool { return t.Name == "CreateError" })).Return(nil, errors.New("failed creating parent")).Once() + + i.MissingRefBehaviour = models.ImportMissingRefEnumCreate + i.Input.Parents = []string{"Create"} + err := i.PostImport(createID) + assert.Nil(t, err) + + i.Input.Parents = []string{"CreateError"} + err = i.PostImport(createErrorID) + assert.NotNil(t, err) + + i.Input.Parents = []string{"CreateFindError"} + err = i.PostImport(createFindErrorID) + assert.NotNil(t, err) + + i.Input.Parents = []string{"CreateFound"} + err = i.PostImport(createFoundID) + assert.Nil(t, err) + + i.MissingRefBehaviour = models.ImportMissingRefEnumFail + i.Input.Parents = []string{"Fail"} + err = i.PostImport(failID) + assert.NotNil(t, err) + + i.Input.Parents = []string{"FailFindError"} + err = i.PostImport(failFindErrorID) + assert.NotNil(t, err) + + i.Input.Parents = []string{"FailFound"} + err = i.PostImport(failFoundID) + assert.Nil(t, err) + + i.MissingRefBehaviour = models.ImportMissingRefEnumIgnore + i.Input.Parents = []string{"Ignore"} + err = i.PostImport(ignoreID) + assert.Nil(t, err) + + i.Input.Parents = []string{"IgnoreFindError"} + err = i.PostImport(ignoreFindErrorID) + assert.NotNil(t, err) + + i.Input.Parents = []string{"IgnoreFound"} + err = i.PostImport(ignoreFoundID) + assert.Nil(t, err) + readerWriter.AssertExpectations(t) } diff --git a/pkg/tag/update.go b/pkg/tag/update.go index 4f5b9b18b..d4a03f8f5 100644 --- a/pkg/tag/update.go +++ b/pkg/tag/update.go @@ -2,7 +2,6 @@ package tag import ( "fmt" - "github.com/stashapp/stash/pkg/models" ) @@ -23,6 +22,20 @@ func (e *NameUsedByAliasError) Error() string { return fmt.Sprintf("name '%s' is used as alias for '%s'", e.Name, e.OtherTag) } +type InvalidTagHierarchyError struct { + Direction string + InvalidTag string + ApplyingTag string +} + +func (e *InvalidTagHierarchyError) Error() string { + if e.InvalidTag == e.ApplyingTag { + return fmt.Sprintf("Cannot apply tag \"%s\" as it already is a %s", e.InvalidTag, e.Direction) + } else { + return fmt.Sprintf("Cannot apply tag \"%s\" as it is linked to \"%s\" which already is a %s", e.ApplyingTag, e.InvalidTag, e.Direction) + } +} + // EnsureTagNameUnique returns an error if the tag name provided // is used as a name or alias of another existing tag. func EnsureTagNameUnique(id int, name string, qb models.TagReader) error { @@ -63,3 +76,150 @@ func EnsureAliasesUnique(id int, aliases []string, qb models.TagReader) error { return nil } + +func EnsureUniqueHierarchy(id int, parentIDs, childIDs []int, qb models.TagReader) error { + allAncestors := make(map[int]*models.Tag) + allDescendants := make(map[int]*models.Tag) + excludeIDs := []int{id} + + validateParent := func(testID, applyingID int) error { + if parentTag, exists := allAncestors[testID]; exists { + applyingTag, err := qb.Find(applyingID) + + if err != nil { + return nil + } + + return &InvalidTagHierarchyError{ + Direction: "parent", + InvalidTag: parentTag.Name, + ApplyingTag: applyingTag.Name, + } + } + + return nil + } + + validateChild := func(testID, applyingID int) error { + if childTag, exists := allDescendants[testID]; exists { + applyingTag, err := qb.Find(applyingID) + + if err != nil { + return nil + } + + return &InvalidTagHierarchyError{ + Direction: "child", + InvalidTag: childTag.Name, + ApplyingTag: applyingTag.Name, + } + } + + return validateParent(testID, applyingID) + } + + if parentIDs == nil { + parentTags, err := qb.FindByChildTagID(id) + if err != nil { + return err + } + + for _, parentTag := range parentTags { + parentIDs = append(parentIDs, parentTag.ID) + } + } + + if childIDs == nil { + childTags, err := qb.FindByParentTagID(id) + if err != nil { + return err + } + + for _, childTag := range childTags { + childIDs = append(childIDs, childTag.ID) + } + } + + for _, parentID := range parentIDs { + parentsAncestors, err := qb.FindAllAncestors(parentID, excludeIDs) + if err != nil { + return err + } + + for _, ancestorTag := range parentsAncestors { + if err := validateParent(ancestorTag.ID, parentID); err != nil { + return err + } + + allAncestors[ancestorTag.ID] = ancestorTag + } + } + + for _, childID := range childIDs { + childsDescendants, err := qb.FindAllDescendants(childID, excludeIDs) + if err != nil { + return err + } + + for _, descendentTag := range childsDescendants { + if err := validateChild(descendentTag.ID, childID); err != nil { + return err + } + + allDescendants[descendentTag.ID] = descendentTag + } + } + + return nil +} + +func MergeHierarchy(destination int, sources []int, qb models.TagReader) ([]int, []int, error) { + var mergedParents, mergedChildren []int + allIds := append([]int{destination}, sources...) + + addTo := func(mergedItems []int, tags []*models.Tag) []int { + Tags: + for _, tag := range tags { + // Ignore tags which are already set + for _, existingItem := range mergedItems { + if tag.ID == existingItem { + continue Tags + } + } + + // Ignore tags which are being merged, as these are rolled up anyway (if A is merged into B any direct link between them can be ignored) + for _, id := range allIds { + if tag.ID == id { + continue Tags + } + } + + mergedItems = append(mergedItems, tag.ID) + } + + return mergedItems + } + + for _, id := range allIds { + parents, err := qb.FindByChildTagID(id) + if err != nil { + return nil, nil, err + } + + mergedParents = addTo(mergedParents, parents) + + children, err := qb.FindByParentTagID(id) + if err != nil { + return nil, nil, err + } + + mergedChildren = addTo(mergedChildren, children) + } + + err := EnsureUniqueHierarchy(destination, mergedParents, mergedChildren, qb) + if err != nil { + return nil, nil, err + } + + return mergedParents, mergedChildren, nil +} diff --git a/pkg/tag/update_test.go b/pkg/tag/update_test.go new file mode 100644 index 000000000..d3a7f226d --- /dev/null +++ b/pkg/tag/update_test.go @@ -0,0 +1,302 @@ +package tag + +import ( + "fmt" + "testing" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var testUniqueHierarchyTags = map[int]*models.Tag{ + 1: { + ID: 1, + Name: "one", + }, + 2: { + ID: 2, + Name: "two", + }, + 3: { + ID: 3, + Name: "three", + }, + 4: { + ID: 4, + Name: "four", + }, +} + +type testUniqueHierarchyCase struct { + id int + parents []*models.Tag + children []*models.Tag + + onFindAllAncestors map[int][]*models.Tag + onFindAllDescendants map[int][]*models.Tag + + expectedError string +} + +var testUniqueHierarchyCases = []testUniqueHierarchyCase{ + { + id: 1, + parents: []*models.Tag{}, + children: []*models.Tag{}, + onFindAllAncestors: map[int][]*models.Tag{ + 1: {}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 1: {}, + }, + expectedError: "", + }, + { + id: 1, + parents: []*models.Tag{testUniqueHierarchyTags[2]}, + children: []*models.Tag{testUniqueHierarchyTags[3]}, + onFindAllAncestors: map[int][]*models.Tag{ + 2: {testUniqueHierarchyTags[2]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3]}, + }, + expectedError: "", + }, + { + id: 2, + parents: []*models.Tag{testUniqueHierarchyTags[3]}, + children: make([]*models.Tag, 0), + onFindAllAncestors: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 2: {testUniqueHierarchyTags[2]}, + }, + expectedError: "", + }, + { + id: 2, + parents: []*models.Tag{ + testUniqueHierarchyTags[3], + testUniqueHierarchyTags[4], + }, + children: []*models.Tag{}, + onFindAllAncestors: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3], testUniqueHierarchyTags[4]}, + 4: {testUniqueHierarchyTags[4]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 2: {testUniqueHierarchyTags[2]}, + }, + expectedError: "Cannot apply tag \"four\" as it already is a parent", + }, + { + id: 2, + parents: []*models.Tag{}, + children: []*models.Tag{testUniqueHierarchyTags[3]}, + onFindAllAncestors: map[int][]*models.Tag{ + 2: {testUniqueHierarchyTags[2]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3]}, + }, + expectedError: "", + }, + { + id: 2, + parents: []*models.Tag{}, + children: []*models.Tag{ + testUniqueHierarchyTags[3], + testUniqueHierarchyTags[4], + }, + onFindAllAncestors: map[int][]*models.Tag{ + 2: {testUniqueHierarchyTags[2]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3], testUniqueHierarchyTags[4]}, + 4: {testUniqueHierarchyTags[4]}, + }, + expectedError: "Cannot apply tag \"four\" as it already is a child", + }, + { + id: 1, + parents: []*models.Tag{testUniqueHierarchyTags[2]}, + children: []*models.Tag{testUniqueHierarchyTags[3]}, + onFindAllAncestors: map[int][]*models.Tag{ + 2: {testUniqueHierarchyTags[2], testUniqueHierarchyTags[3]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3]}, + }, + expectedError: "Cannot apply tag \"three\" as it already is a parent", + }, + { + id: 1, + parents: []*models.Tag{testUniqueHierarchyTags[2]}, + children: []*models.Tag{testUniqueHierarchyTags[3]}, + onFindAllAncestors: map[int][]*models.Tag{ + 2: {testUniqueHierarchyTags[2]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3], testUniqueHierarchyTags[2]}, + }, + expectedError: "Cannot apply tag \"three\" as it is linked to \"two\" which already is a parent", + }, + { + id: 1, + parents: []*models.Tag{testUniqueHierarchyTags[3]}, + children: []*models.Tag{testUniqueHierarchyTags[3]}, + onFindAllAncestors: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3]}, + }, + expectedError: "Cannot apply tag \"three\" as it already is a parent", + }, + { + id: 1, + parents: []*models.Tag{ + testUniqueHierarchyTags[2], + }, + children: []*models.Tag{ + testUniqueHierarchyTags[3], + }, + onFindAllAncestors: map[int][]*models.Tag{ + 2: {testUniqueHierarchyTags[2]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3], testUniqueHierarchyTags[2]}, + }, + expectedError: "Cannot apply tag \"three\" as it is linked to \"two\" which already is a parent", + }, + { + id: 1, + parents: []*models.Tag{testUniqueHierarchyTags[2]}, + children: []*models.Tag{testUniqueHierarchyTags[2]}, + onFindAllAncestors: map[int][]*models.Tag{ + 2: {testUniqueHierarchyTags[2]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 2: {testUniqueHierarchyTags[2]}, + }, + expectedError: "Cannot apply tag \"two\" as it already is a parent", + }, + { + id: 2, + parents: []*models.Tag{testUniqueHierarchyTags[1]}, + children: []*models.Tag{testUniqueHierarchyTags[3]}, + onFindAllAncestors: map[int][]*models.Tag{ + 1: {testUniqueHierarchyTags[1]}, + }, + onFindAllDescendants: map[int][]*models.Tag{ + 3: {testUniqueHierarchyTags[3], testUniqueHierarchyTags[1]}, + }, + expectedError: "Cannot apply tag \"three\" as it is linked to \"one\" which already is a parent", + }, +} + +func TestEnsureUniqueHierarchy(t *testing.T) { + for _, tc := range testUniqueHierarchyCases { + testEnsureUniqueHierarchy(t, tc, false, false) + testEnsureUniqueHierarchy(t, tc, true, false) + testEnsureUniqueHierarchy(t, tc, false, true) + testEnsureUniqueHierarchy(t, tc, true, true) + } +} + +func testEnsureUniqueHierarchy(t *testing.T, tc testUniqueHierarchyCase, queryParents, queryChildren bool) { + mockTagReader := &mocks.TagReaderWriter{} + + var parentIDs, childIDs []int + find := make(map[int]*models.Tag) + if tc.parents != nil { + parentIDs = make([]int, 0) + for _, parent := range tc.parents { + if parent.ID != tc.id { + find[parent.ID] = parent + parentIDs = append(parentIDs, parent.ID) + } + } + } + + if tc.children != nil { + childIDs = make([]int, 0) + for _, child := range tc.children { + if child.ID != tc.id { + find[child.ID] = child + childIDs = append(childIDs, child.ID) + } + } + } + + if queryParents { + parentIDs = nil + mockTagReader.On("FindByChildTagID", tc.id).Return(tc.parents, nil).Once() + } + + if queryChildren { + childIDs = nil + mockTagReader.On("FindByParentTagID", tc.id).Return(tc.children, nil).Once() + } + + mockTagReader.On("Find", mock.AnythingOfType("int")).Return(func(tagID int) *models.Tag { + for id, tag := range find { + if id == tagID { + return tag + } + } + return nil + }, func(tagID int) error { + return nil + }).Maybe() + + mockTagReader.On("FindAllAncestors", mock.AnythingOfType("int"), []int{tc.id}).Return(func(tagID int, excludeIDs []int) []*models.Tag { + for id, tags := range tc.onFindAllAncestors { + if id == tagID { + return tags + } + } + return nil + }, func(tagID int, excludeIDs []int) error { + for id := range tc.onFindAllAncestors { + if id == tagID { + return nil + } + } + return fmt.Errorf("undefined ancestors for: %d", tagID) + }).Maybe() + + mockTagReader.On("FindAllDescendants", mock.AnythingOfType("int"), []int{tc.id}).Return(func(tagID int, excludeIDs []int) []*models.Tag { + for id, tags := range tc.onFindAllDescendants { + if id == tagID { + return tags + } + } + return nil + }, func(tagID int, excludeIDs []int) error { + for id := range tc.onFindAllDescendants { + if id == tagID { + return nil + } + } + return fmt.Errorf("undefined descendants for: %d", tagID) + }).Maybe() + + res := EnsureUniqueHierarchy(tc.id, parentIDs, childIDs, mockTagReader) + + assert := assert.New(t) + + if tc.expectedError != "" { + if assert.NotNil(res) { + assert.Equal(tc.expectedError, res.Error()) + } + } else { + assert.Nil(res) + } + + mockTagReader.AssertExpectations(t) +} diff --git a/pkg/utils/boolean.go b/pkg/utils/boolean.go index a5f23733b..80fa77ec7 100644 --- a/pkg/utils/boolean.go +++ b/pkg/utils/boolean.go @@ -1,13 +1,5 @@ package utils -// Btoi transforms a boolean to an int. 1 for true, false otherwise -func Btoi(b bool) int { - if b { - return 1 - } - return 0 -} - // IsTrue returns true if the bool pointer is not nil and true. func IsTrue(b *bool) bool { return b != nil && *b diff --git a/pkg/utils/byterange.go b/pkg/utils/byterange.go index 2220b8dc4..5fb031ea7 100644 --- a/pkg/utils/byterange.go +++ b/pkg/utils/byterange.go @@ -30,14 +30,6 @@ func CreateByteRange(s string) ByteRange { return ret } -func (r ByteRange) getBytesToRead() int64 { - if r.End == nil { - return 0 - } - - return *r.End - r.Start + 1 -} - func (r ByteRange) ToHeaderValue(fileLength int64) string { if r.End == nil { return "" diff --git a/pkg/utils/crypto.go b/pkg/utils/crypto.go index 8f20ad9b2..6022aeac8 100644 --- a/pkg/utils/crypto.go +++ b/pkg/utils/crypto.go @@ -7,6 +7,8 @@ import ( "hash/fnv" "io" "os" + + "github.com/stashapp/stash/pkg/logger" ) func MD5FromBytes(data []byte) string { @@ -40,7 +42,9 @@ func MD5FromReader(src io.Reader) (string, error) { func GenerateRandomKey(l int) string { b := make([]byte, l) - rand.Read(b) + if n, err := rand.Read(b); err != nil { + logger.Warnf("failure generating random key: %v (only read %v bytes)", err, n) + } return fmt.Sprintf("%x", b) } diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 87f7538b3..722fc1f88 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -4,7 +4,6 @@ import ( "archive/zip" "fmt" "io" - "io/ioutil" "net/http" "os" "os/user" @@ -108,10 +107,10 @@ func EmptyDir(path string) error { // ListDir will return the contents of a given directory path as a string slice func ListDir(path string) ([]string, error) { var dirPaths []string - files, err := ioutil.ReadDir(path) + files, err := os.ReadDir(path) if err != nil { path = filepath.Dir(path) - files, err = ioutil.ReadDir(path) + files, err = os.ReadDir(path) if err != nil { return dirPaths, err } @@ -193,12 +192,12 @@ func IsZipFileUncompressed(path string) (bool, error) { func WriteFile(path string, file []byte) error { pathErr := EnsureDirAll(filepath.Dir(path)) if pathErr != nil { - return fmt.Errorf("Cannot ensure path %s", pathErr) + return fmt.Errorf("cannot ensure path %s", pathErr) } - err := ioutil.WriteFile(path, file, 0755) + err := os.WriteFile(path, file, 0755) if err != nil { - return fmt.Errorf("Write error for thumbnail %s: %s ", path, err) + return fmt.Errorf("write error for thumbnail %s: %s ", path, err) } return nil } diff --git a/pkg/utils/file_test.go b/pkg/utils/file_test.go index 869cd2af6..4be89c3e4 100644 --- a/pkg/utils/file_test.go +++ b/pkg/utils/file_test.go @@ -1,7 +1,6 @@ package utils import ( - "io/ioutil" "os" "path/filepath" "testing" @@ -53,11 +52,11 @@ func TestDirExists(t *testing.T) { const st = "stash_tmp" tmp := os.TempDir() - tmpDir, err := ioutil.TempDir(tmp, st) // create a tmp dir in the system's tmp folder + tmpDir, err := os.MkdirTemp(tmp, st) // create a tmp dir in the system's tmp folder if err == nil { defer os.RemoveAll(tmpDir) - tmpFile, err := ioutil.TempFile(tmpDir, st) + tmpFile, err := os.CreateTemp(tmpDir, st) if err != nil { return } diff --git a/pkg/utils/image.go b/pkg/utils/image.go index ad4f56941..75903fa6d 100644 --- a/pkg/utils/image.go +++ b/pkg/utils/image.go @@ -5,7 +5,7 @@ import ( "crypto/tls" "encoding/base64" "fmt" - "io/ioutil" + "io" "net/http" "regexp" "strings" @@ -66,7 +66,7 @@ func ReadImageFromURL(url string) ([]byte, error) { defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/pkg/utils/oshash_internal_test.go b/pkg/utils/oshash_internal_test.go index 0263bab8a..b4d5b9d35 100644 --- a/pkg/utils/oshash_internal_test.go +++ b/pkg/utils/oshash_internal_test.go @@ -12,7 +12,7 @@ func TestOshashEmpty(t *testing.T) { want := "0000000000000000" got, err := oshash(size, head, tail) if err != nil { - t.Errorf("TestOshashEmpty: Error from oshash: %w", err) + t.Errorf("TestOshashEmpty: Error from oshash: %v", err) } if got != want { t.Errorf("TestOshashEmpty: oshash(0, 0, 0) = %q; want %q", got, want) diff --git a/revive.toml b/revive.toml deleted file mode 100644 index c535a9a5c..000000000 --- a/revive.toml +++ /dev/null @@ -1,30 +0,0 @@ -ignoreGeneratedHeader = false -severity = "error" -confidence = 0.8 -errorCode = 1 -warningCode = 1 - -#[rule.blank-imports] -[rule.context-as-argument] -[rule.context-keys-type] -[rule.dot-imports] -[rule.error-return] -[rule.error-strings] -[rule.error-naming] -#[rule.exported] -#[rule.if-return] -[rule.increment-decrement] -#[rule.var-naming] -[rule.var-declaration] -[rule.package-comments] -[rule.range] -[rule.receiver-naming] -[rule.time-naming] -#[rule.unexported-return] -#[rule.indent-error-flow] -[rule.errorf] -#[rule.empty-block] -[rule.superfluous-else] -#[rule.unused-parameter] -[rule.unreachable-code] -[rule.redefines-builtin-id] \ No newline at end of file diff --git a/scripts/check-gofmt.sh b/scripts/check-gofmt.sh deleted file mode 100644 index 24fa914a5..000000000 --- a/scripts/check-gofmt.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh - -# Copyright (c) 2012 The Go Authors. All rights reserved. - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: - -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -gofiles=./pkg/ -[ "$OS" = "Windows_NT" ] && gofiles=.\\pkg\\ - -unformatted=$(gofmt -l $gofiles) -[ -z "$unformatted" ] && exit 0 - -# Some files are not gofmt'd. Print message and fail. - -echo >&2 "Go files must be formatted with gofmt. Please run:" -for fn in $unformatted; do - echo >&2 " gofmt -w $PWD/$fn" -done - -exit 1 diff --git a/scripts/cross-compile.sh b/scripts/cross-compile.sh index e46abed28..2012d9b6b 100755 --- a/scripts/cross-compile.sh +++ b/scripts/cross-compile.sh @@ -1,31 +1,23 @@ #!/bin/bash # "stashapp/compiler:develop" "stashapp/compiler:4" -COMPILER_CONTAINER="stashapp/compiler:4" +COMPILER_CONTAINER="stashapp/compiler:5" BUILD_DATE=`go run -mod=vendor scripts/getDate.go` GITHASH=`git rev-parse --short HEAD` STASH_VERSION=`git describe --tags --exclude latest_develop` SETENV="BUILD_DATE=\"$BUILD_DATE\" GITHASH=$GITHASH STASH_VERSION=\"$STASH_VERSION\"" -SETUP="export GO111MODULE=on; export CGO_ENABLED=1; set -e; echo '=== Running packr ==='; make packr;" -SETUP_FAST="export GO111MODULE=on; export CGO_ENABLED=1;" +SETUP="export CGO_ENABLED=1;" WINDOWS="echo '=== Building Windows binary ==='; $SETENV make cross-compile-windows;" -DARWIN="echo '=== Building OSX binary ==='; $SETENV make cross-compile-osx;" +DARWIN="echo '=== Building OSX binary ==='; $SETENV make cross-compile-osx-intel;" +DARWIN_ARM64="echo '=== Building OSX (arm64) binary ==='; $SETENV make cross-compile-osx-applesilicon;" LINUX_AMD64="echo '=== Building Linux (amd64) binary ==='; $SETENV make cross-compile-linux;" LINUX_ARM64v8="echo '=== Building Linux (armv8/arm64) binary ==='; $SETENV make cross-compile-linux-arm64v8;" LINUX_ARM32v7="echo '=== Building Linux (armv7/armhf) binary ==='; $SETENV make cross-compile-linux-arm32v7;" LINUX_ARM32v6="echo '=== Building Linux (armv6 | Raspberry Pi 1) binary ==='; $SETENV make cross-compile-pi;" BUILD_COMPLETE="echo '=== Build complete ==='" -# if build target ends with -fast then use prebuilt packr2. eg amd64-fast or all-fast -FAST=`echo "$1" | cut -d - -f 2` -if [ "$FAST" == "fast" ] -then - echo "Building without Packr2" - SETUP=$SETUP_FAST -fi - BUILD=`echo "$1" | cut -d - -f 1` if [ "$BUILD" == "windows" ] then @@ -53,7 +45,7 @@ then COMMAND="$SETUP $LINUX_ARM32v7 $BUILD_COMPLETE" else echo "Building All" - COMMAND="$SETUP $WINDOWS $DARWIN $LINUX_AMD64 $LINUX_ARM64v8 $LINUX_ARM32v7 $LINUX_ARM32v6 $BUILD_COMPLETE" + COMMAND="$SETUP $WINDOWS $DARWIN $DARWIN_ARM64 $LINUX_AMD64 $LINUX_ARM64v8 $LINUX_ARM32v7 $LINUX_ARM32v6 $BUILD_COMPLETE" fi # Pull Latest Image diff --git a/scripts/test_db_generator/config.yml b/scripts/test_db_generator/config.yml index d1870b45c..ac072c459 100644 --- a/scripts/test_db_generator/config.yml +++ b/scripts/test_db_generator/config.yml @@ -1,10 +1,10 @@ database: generated.sqlite scenes: 30000 -images: 150000 +images: 4000000 galleries: 1500 -markers: 300 +markers: 3000 performers: 10000 -studios: 500 +studios: 1500 tags: 1500 naming: scenes: scene.txt diff --git a/scripts/test_db_generator/makeTestDB.go b/scripts/test_db_generator/makeTestDB.go index b7b35089d..e3aa59033 100644 --- a/scripts/test_db_generator/makeTestDB.go +++ b/scripts/test_db_generator/makeTestDB.go @@ -1,4 +1,4 @@ -// uild ignore +// +build ignore package main @@ -6,6 +6,8 @@ import ( "context" "database/sql" "fmt" + "log" + "math" "math/rand" "os" "strconv" @@ -19,7 +21,7 @@ import ( "gopkg.in/yaml.v2" ) -const batchSize = 1000 +const batchSize = 50000 // create an example database by generating a number of scenes, markers, // performers, studios and tags, and associating between them all @@ -40,15 +42,19 @@ var txnManager models.TransactionManager var c *config func main() { + rand.Seed(time.Now().UnixNano()) + var err error c, err = loadConfig() if err != nil { - panic(err) + log.Fatalf("couldn't load configuration: %v", err) } initNaming(*c) - database.Initialize(c.Database) + if err = database.Initialize(c.Database); err != nil { + log.Fatalf("couldn't initialize database: %v", err) + } populateDB() } @@ -78,6 +84,7 @@ func populateDB() { makeScenes(c.Scenes) makeImages(c.Images) makeGalleries(c.Galleries) + makeMarkers(c.Markers) } func withTxn(f func(r models.Repository) error) error { @@ -109,8 +116,25 @@ func makeTags(n int) { Name: name, } - _, err := r.Tag().Create(tag) - return err + created, err := r.Tag().Create(tag) + if err != nil { + return err + } + + if rand.Intn(100) > 5 { + t, _, err := r.Tag().Query(nil, getRandomFilter(1)) + if err != nil { + return err + } + + if len(t) > 0 && t[0].ID != created.ID { + if err := r.Tag().UpdateParentTags(created.ID, []int{t[0].ID}); err != nil { + return err + } + } + } + + return nil }) }); err != nil { panic(err) @@ -181,7 +205,6 @@ func makePerformers(n int) { func makeScenes(n int) { logger.Infof("creating %d scenes...", n) - rand.Seed(533) for i := 0; i < n; { // do in batches of 1000 batch := i + batchSize @@ -256,7 +279,6 @@ func generateScene(i int) models.Scene { func makeImages(n int) { logger.Infof("creating %d images...", n) - rand.Seed(1293) for i := 0; i < n; { // do in batches of 1000 batch := i + batchSize @@ -298,7 +320,6 @@ func generateImage(i int) models.Image { func makeGalleries(n int) { logger.Infof("creating %d galleries...", n) - rand.Seed(92113) for i := 0; i < n; { // do in batches of 1000 batch := i + batchSize @@ -339,8 +360,48 @@ func generateGallery(i int) models.Gallery { } } +func makeMarkers(n int) { + logger.Infof("creating %d markers...", n) + for i := 0; i < n; { + // do in batches of 1000 + batch := i + batchSize + if err := withTxn(func(r models.Repository) error { + for ; i < batch && i < n; i++ { + marker := generateMarker(i) + marker.SceneID = models.NullInt64(int64(getRandomScene())) + marker.PrimaryTagID = getRandomTags(r, 1, 1)[0] + + created, err := r.SceneMarker().Create(marker) + if err != nil { + return err + } + + tags := getRandomTags(r, 0, 5) + // remove primary tag + tags = utils.IntExclude(tags, []int{marker.PrimaryTagID}) + if err := r.SceneMarker().UpdateTags(created.ID, tags); err != nil { + return err + } + } + + logger.Infof("... created %d markers", i) + + return nil + }); err != nil { + panic(err) + } + } +} + +func generateMarker(i int) models.SceneMarker { + return models.SceneMarker{ + Title: names[c.Naming.Scenes].generateName(rand.Intn(7) + 1), + } +} + func getRandomFilter(n int) *models.FindFilterType { - sortBy := "random" + seed := math.Floor(rand.Float64() * math.Pow10(8)) + sortBy := fmt.Sprintf("random_%.f", seed) return &models.FindFilterType{ Sort: &sortBy, PerPage: &n, @@ -365,7 +426,7 @@ func getRandomStudioID(r models.Repository) sql.NullInt64 { func makeSceneRelationships(r models.Repository, id int) { // add tags - tagIDs := getRandomTags(r) + tagIDs := getRandomTags(r, 0, 15) if len(tagIDs) > 0 { if err := r.Scene().UpdateTags(id, tagIDs); err != nil { panic(err) @@ -382,26 +443,33 @@ func makeSceneRelationships(r models.Repository, id int) { } func makeImageRelationships(r models.Repository, id int) { + // there are typically many more images. For performance reasons + // only a small proportion should have tags/performers + // add tags - tagIDs := getRandomTags(r) - if len(tagIDs) > 0 { - if err := r.Image().UpdateTags(id, tagIDs); err != nil { - panic(err) + if rand.Intn(100) == 0 { + tagIDs := getRandomTags(r, 1, 15) + if len(tagIDs) > 0 { + if err := r.Image().UpdateTags(id, tagIDs); err != nil { + panic(err) + } } } // add performers - performerIDs := getRandomPerformers(r) - if len(tagIDs) > 0 { - if err := r.Image().UpdatePerformers(id, performerIDs); err != nil { - panic(err) + if rand.Intn(100) <= 1 { + performerIDs := getRandomPerformers(r) + if len(performerIDs) > 0 { + if err := r.Image().UpdatePerformers(id, performerIDs); err != nil { + panic(err) + } } } } func makeGalleryRelationships(r models.Repository, id int) { // add tags - tagIDs := getRandomTags(r) + tagIDs := getRandomTags(r, 0, 15) if len(tagIDs) > 0 { if err := r.Gallery().UpdateTags(id, tagIDs); err != nil { panic(err) @@ -447,8 +515,17 @@ func getRandomPerformers(r models.Repository) []int { return ret } -func getRandomTags(r models.Repository) []int { - n := rand.Intn(15) +func getRandomScene() int { + return rand.Intn(c.Scenes) + 1 +} + +func getRandomTags(r models.Repository, min, max int) []int { + var n int + if min == max { + n = min + } else { + n = rand.Intn(max-min) + min + } var ret []int // if n > 0 { diff --git a/scripts/test_db_generator/naming.go b/scripts/test_db_generator/naming.go index 90f68d8a5..cc016dce5 100644 --- a/scripts/test_db_generator/naming.go +++ b/scripts/test_db_generator/naming.go @@ -1,3 +1,5 @@ +// +build ignore + package main import ( diff --git a/tools.go b/tools.go index 9bc7b212d..fa219bb53 100644 --- a/tools.go +++ b/tools.go @@ -1,3 +1,4 @@ +//go:build tools // +build tools package main diff --git a/ui/login/login.html b/ui/login/login.html index a7c06595d..b23a8b77c 100644 --- a/ui/login/login.html +++ b/ui/login/login.html @@ -1,18 +1,19 @@ + Login - - + +
-
+
@@ -36,4 +37,4 @@
- \ No newline at end of file + diff --git a/ui/v2.5/package.json b/ui/v2.5/package.json index c36fd1989..3eb6464c7 100644 --- a/ui/v2.5/package.json +++ b/ui/v2.5/package.json @@ -2,6 +2,7 @@ "name": "stash", "version": "0.1.0", "private": true, + "homepage": "./", "sideEffects": false, "scripts": { "start": "react-scripts start", @@ -36,7 +37,7 @@ "@fortawesome/react-fontawesome": "^0.1.14", "@types/react-select": "^4.0.8", "apollo-upload-client": "^14.1.3", - "axios": "0.21.1", + "axios": "0.21.2", "base64-blob": "^1.4.1", "bootstrap": "^4.6.0", "classnames": "^2.2.6", diff --git a/ui/v2.5/public/index.html b/ui/v2.5/public/index.html index 322bc1d11..62c7d3940 100755 --- a/ui/v2.5/public/index.html +++ b/ui/v2.5/public/index.html @@ -1,6 +1,7 @@ + Stash + diff --git a/ui/v2.5/src/components/Changelog/Changelog.tsx b/ui/v2.5/src/components/Changelog/Changelog.tsx index 87b8ff1db..f504f81de 100644 --- a/ui/v2.5/src/components/Changelog/Changelog.tsx +++ b/ui/v2.5/src/components/Changelog/Changelog.tsx @@ -12,6 +12,7 @@ import V060 from "./versions/v060.md"; import V070 from "./versions/v070.md"; import V080 from "./versions/v080.md"; import V090 from "./versions/v090.md"; +import V0100 from "./versions/v0100.md"; import { MarkdownPage } from "../Shared/MarkdownPage"; // to avoid use of explicit any @@ -50,9 +51,9 @@ const Changelog: React.FC = () => { // after new release: // add entry to releases, using the current* fields // then update the current fields. - const currentVersion = stashVersion || "v0.9.0"; + const currentVersion = stashVersion || "v0.10.0"; const currentDate = buildDate; - const currentPage = V090; + const currentPage = V0100; const releases: IStashRelease[] = [ { @@ -61,6 +62,11 @@ const Changelog: React.FC = () => { page: currentPage, defaultOpen: true, }, + { + version: "v0.9.0", + date: "2021-09-06", + page: V090, + }, { version: "v0.8.0", date: "2021-07-02", diff --git a/ui/v2.5/src/components/Changelog/versions/v0100.md b/ui/v2.5/src/components/Changelog/versions/v0100.md new file mode 100644 index 000000000..79789dd13 --- /dev/null +++ b/ui/v2.5/src/components/Changelog/versions/v0100.md @@ -0,0 +1,43 @@ +#### 💥 Note: Please check your logs after migrating to this release. A log warning will be generated on startup if duplicate image checksums exist in your system. Search for the images using the logged checksums, and remove the unwanted ones. + +#### 💥 Note: The system will now stop serving requests if authentication is not configured and it detects a connection from public internet. See [this link](https://github.com/stashapp/stash/wiki/Authentication-Required-When-Accessing-Stash-From-the-Internet) for details. + +### ✨ New Features +* Added support for Tag hierarchies. ([#1519](https://github.com/stashapp/stash/pull/1519)) +* Revamped image lightbox to support zoom, pan and various display modes. ([#1708](https://github.com/stashapp/stash/pull/1708)) +* Added support for Studio aliases. ([#1660](https://github.com/stashapp/stash/pull/1660)) +* Added support for querying scene scrapers using keywords. ([#1712](https://github.com/stashapp/stash/pull/1712)) +* Added native support for Apple Silicon / M1 Macs. ([#1646](https://github.com/stashapp/stash/pull/1646)) +* Support subpaths when serving stash via reverse proxy. ([#1719](https://github.com/stashapp/stash/pull/1719)) +* Disallow access from public internet addresses when authentication is not configured. ([#1761](https://github.com/stashapp/stash/pull/1761)) +* Added options to generate webp and static preview files for markers. ([#1604](https://github.com/stashapp/stash/pull/1604)) +* Added sort by option for gallery rating. ([#1720](https://github.com/stashapp/stash/pull/1720)) +* Added Movies to Scene bulk edit dialog. ([#1676](https://github.com/stashapp/stash/pull/1676)) +* Added Movies tab to Studio and Performer pages. ([#1675](https://github.com/stashapp/stash/pull/1675)) +* Support filtering Movies by Performers. ([#1675](https://github.com/stashapp/stash/pull/1675)) + +### 🎨 Improvements +* Optimised image thumbnail generation (optionally using `libvips`) and made optional. ([#1655](https://github.com/stashapp/stash/pull/1655)) +* Improved image query performance. ([#1740](https://github.com/stashapp/stash/pull/1740), [#1750](https://github.com/stashapp/stash/pull/1750)) +* Support setting metadata import/export directory from UI. ([#1782](https://github.com/stashapp/stash/pull/1782)) +* Added movie count to performer and studio cards. ([#1760](https://github.com/stashapp/stash/pull/1760)) +* Added date and details to Movie card, and move scene count to icon. ([#1758](https://github.com/stashapp/stash/pull/1758)) +* Added date and details to Gallery card, and move image count to icon. ([#1763](https://github.com/stashapp/stash/pull/1763)) +* Support scraper script logging to specific log levels. ([#1648](https://github.com/stashapp/stash/pull/1648)) +* Added sv-SE language option. ([#1691](https://github.com/stashapp/stash/pull/1691)) + +### 🐛 Bug fixes +* Disabled float-on-scroll player on mobile devices. ([#1721](https://github.com/stashapp/stash/pull/1721)) +* Fix video transcoding process starting before video is played. ([#1780](https://github.com/stashapp/stash/pull/1780)) +* Fix Scene Edit Panel form layout for mobile and desktop. ([#1737](https://github.com/stashapp/stash/pull/1737)) +* Don't scan zero-length files. ([#1779](https://github.com/stashapp/stash/pull/1779)) +* Accept svg files in file selector for tag images. ([#1778](https://github.com/stashapp/stash/pull/1778)) +* Optimised exclude filter queries. ([#1815](https://github.com/stashapp/stash/pull/1815)) +* Fix video player aspect ratio shifting sometimes when clicking scene tabs. ([#1764](https://github.com/stashapp/stash/pull/1764)) +* Fix criteria being incorrectly applied when clicking back button. ([#1765](https://github.com/stashapp/stash/pull/1765)) +* Show first page and fix order direction not being maintained when clicking on card popover button. ([#1765](https://github.com/stashapp/stash/pull/1765)) +* Fix panic in autotagger when backslash character present in tag/performer/studio name. ([#1753](https://github.com/stashapp/stash/pull/1753)) +* Fix Scene Player CLS issue ([#1739](https://github.com/stashapp/stash/pull/1739)) +* Fix Gallery create plugin hook not being invoked when creating Gallery from folder. ([#1731](https://github.com/stashapp/stash/pull/1731)) +* Fix tag aliases not being matched when autotagging from the tasks page. ([#1713](https://github.com/stashapp/stash/pull/1713)) +* Fix Create Marker form on small devices. ([#1718](https://github.com/stashapp/stash/pull/1718)) diff --git a/ui/v2.5/src/components/Galleries/GalleryCard.tsx b/ui/v2.5/src/components/Galleries/GalleryCard.tsx index 857ca7f0c..4f9a9ade0 100644 --- a/ui/v2.5/src/components/Galleries/GalleryCard.tsx +++ b/ui/v2.5/src/components/Galleries/GalleryCard.tsx @@ -2,10 +2,16 @@ import { Button, ButtonGroup } from "react-bootstrap"; import React from "react"; import { Link } from "react-router-dom"; import * as GQL from "src/core/generated-graphql"; -import { FormattedPlural } from "react-intl"; import { useConfiguration } from "src/core/StashService"; -import { GridCard, HoverPopover, Icon, TagLink } from "src/components/Shared"; -import { TextUtils } from "src/utils"; +import { + GridCard, + HoverPopover, + Icon, + TagLink, + TruncatedText, +} from "src/components/Shared"; +import { PopoverCountButton } from "src/components/Shared/PopoverCountButton"; +import { NavUtils, TextUtils } from "src/utils"; import { PerformerPopoverButton } from "../Shared/PerformerPopoverButton"; import { RatingBanner } from "../Shared/RatingBanner"; @@ -30,7 +36,11 @@ export const GalleryCard: React.FC = (props) => { )); return ( - + + + ); + } + + function maybeRenderPopoverButtonGroup() { + if (props.sceneIndex || props.movie.scenes.length > 0) { + return ( + <> + {maybeRenderSceneNumber()} +
+ + {maybeRenderScenesPopoverButton()} + + ); } - - return Scene number: {props.sceneIndex}; } return ( @@ -46,10 +84,18 @@ export const MovieCard: FunctionComponent = (props: IProps) => { } - details={maybeRenderSceneNumber()} + details={ + <> + {props.movie.date} +

+ +

+ + } selected={props.selected} selecting={props.selecting} onSelectedChanged={props.onSelectedChanged} + popovers={maybeRenderPopoverButtonGroup()} /> ); }; diff --git a/ui/v2.5/src/components/Movies/MovieDetails/MovieEditPanel.tsx b/ui/v2.5/src/components/Movies/MovieDetails/MovieEditPanel.tsx index 24d7f3af1..981d52751 100644 --- a/ui/v2.5/src/components/Movies/MovieDetails/MovieEditPanel.tsx +++ b/ui/v2.5/src/components/Movies/MovieDetails/MovieEditPanel.tsx @@ -203,8 +203,8 @@ export const MovieEditPanel: React.FC = ({ formik.setFieldValue("date", state.date ?? undefined); } - if (state.studio && state.studio.id) { - formik.setFieldValue("studio_id", state.studio.id ?? undefined); + if (state.studio && state.studio.stored_id) { + formik.setFieldValue("studio_id", state.studio.stored_id ?? undefined); } if (state.director) { diff --git a/ui/v2.5/src/components/Movies/MovieDetails/MovieScrapeDialog.tsx b/ui/v2.5/src/components/Movies/MovieDetails/MovieScrapeDialog.tsx index aad6735b6..467fa96a7 100644 --- a/ui/v2.5/src/components/Movies/MovieDetails/MovieScrapeDialog.tsx +++ b/ui/v2.5/src/components/Movies/MovieDetails/MovieScrapeDialog.tsx @@ -87,7 +87,10 @@ export const MovieScrapeDialog: React.FC = ( new ScrapeResult(props.movie.synopsis, props.scraped.synopsis) ); const [studio, setStudio] = useState>( - new ScrapeResult(props.movie.studio_id, props.scraped.studio?.id) + new ScrapeResult( + props.movie.studio_id, + props.scraped.studio?.stored_id + ) ); const [url, setURL] = useState>( new ScrapeResult(props.movie.url, props.scraped.url) @@ -123,7 +126,7 @@ export const MovieScrapeDialog: React.FC = ( const durationString = duration.getNewValue(); return { - name: name.getNewValue(), + name: name.getNewValue() ?? "", aliases: aliases.getNewValue(), duration: durationString, date: date.getNewValue(), @@ -131,7 +134,7 @@ export const MovieScrapeDialog: React.FC = ( synopsis: synopsis.getNewValue(), studio: newStudio ? { - id: newStudio, + stored_id: newStudio, name: "", } : undefined, diff --git a/ui/v2.5/src/components/Movies/MovieList.tsx b/ui/v2.5/src/components/Movies/MovieList.tsx index 4329e9af3..ac5b3a5bc 100644 --- a/ui/v2.5/src/components/Movies/MovieList.tsx +++ b/ui/v2.5/src/components/Movies/MovieList.tsx @@ -18,7 +18,11 @@ import { import { ExportDialog, DeleteEntityDialog } from "src/components/Shared"; import { MovieCard } from "./MovieCard"; -export const MovieList: React.FC = () => { +interface IMovieList { + filterHook?: (filter: ListFilterModel) => ListFilterModel; +} + +export const MovieList: React.FC = ({ filterHook }) => { const intl = useIntl(); const history = useHistory(); const [isExportDialogOpen, setIsExportDialogOpen] = useState(false); @@ -73,6 +77,7 @@ export const MovieList: React.FC = () => { selectable: true, persistState: PersistanceLevel.ALL, renderDeleteDialog, + filterHook, }); async function viewRandom( diff --git a/ui/v2.5/src/components/Movies/styles.scss b/ui/v2.5/src/components/Movies/styles.scss index c87122a55..3452b265f 100644 --- a/ui/v2.5/src/components/Movies/styles.scss +++ b/ui/v2.5/src/components/Movies/styles.scss @@ -17,6 +17,10 @@ object-fit: contain; width: 100%; } + + .movie-scene-number { + text-align: center; + } } .movie-images { diff --git a/ui/v2.5/src/components/Performers/PerformerCard.tsx b/ui/v2.5/src/components/Performers/PerformerCard.tsx index 4e2affda6..619511e70 100644 --- a/ui/v2.5/src/components/Performers/PerformerCard.tsx +++ b/ui/v2.5/src/components/Performers/PerformerCard.tsx @@ -21,6 +21,7 @@ export interface IPerformerCardExtraCriteria { scenes: Criterion[]; images: Criterion[]; galleries: Criterion[]; + movies: Criterion[]; } interface IPerformerCardProps { @@ -73,6 +74,7 @@ export const PerformerCard: React.FC = ({ return ( = ({ return ( = ({ return ( = ({ return ( - @@ -124,18 +128,33 @@ export const PerformerCard: React.FC = ({ ); } + function maybeRenderMoviesPopoverButton() { + if (!performer.movie_count) return; + + return ( + + ); + } + function maybeRenderPopoverButtonGroup() { if ( performer.scene_count || performer.image_count || performer.gallery_count || - performer.tags.length > 0 + performer.tags.length > 0 || + performer.movie_count ) { return ( <>
{maybeRenderScenesPopoverButton()} + {maybeRenderMoviesPopoverButton()} {maybeRenderImagesPopoverButton()} {maybeRenderGalleriesPopoverButton()} {maybeRenderTagPopoverButton()} diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx index 5297e74f9..92f602679 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx @@ -22,6 +22,7 @@ import { PerformerDetailsPanel } from "./PerformerDetailsPanel"; import { PerformerOperationsPanel } from "./PerformerOperationsPanel"; import { PerformerScenesPanel } from "./PerformerScenesPanel"; import { PerformerGalleriesPanel } from "./PerformerGalleriesPanel"; +import { PerformerMoviesPanel } from "./PerformerMoviesPanel"; import { PerformerImagesPanel } from "./PerformerImagesPanel"; import { PerformerEditPanel } from "./PerformerEditPanel"; @@ -70,6 +71,7 @@ export const Performer: React.FC = () => { tab === "scenes" || tab === "galleries" || tab === "images" || + tab === "movies" || tab === "edit" || tab === "operations" ? tab @@ -91,6 +93,7 @@ export const Performer: React.FC = () => { Mousetrap.bind("e", () => setActiveTabKey("edit")); Mousetrap.bind("c", () => setActiveTabKey("scenes")); Mousetrap.bind("g", () => setActiveTabKey("galleries")); + Mousetrap.bind("m", () => setActiveTabKey("movies")); Mousetrap.bind("o", () => setActiveTabKey("operations")); Mousetrap.bind("f", () => setFavorite(!performer.favorite)); @@ -140,6 +143,9 @@ export const Performer: React.FC = () => { + + + = ({ // Editing state const [scraper, setScraper] = useState(); - const [newTags, setNewTags] = useState(); + const [newTags, setNewTags] = useState(); const [isScraperModalOpen, setIsScraperModalOpen] = useState(false); const [isDeleteAlertOpen, setIsDeleteAlertOpen] = useState(false); @@ -224,7 +224,7 @@ export const PerformerEditPanel: React.FC = ({ return ret; } - async function createNewTag(toCreate: GQL.ScrapedSceneTag) { + async function createNewTag(toCreate: GQL.ScrapedTag) { const tagInput: GQL.TagCreateInput = { name: toCreate.name ?? "" }; try { const result = await createTag({ @@ -334,9 +334,10 @@ export const PerformerEditPanel: React.FC = ({ // otherwise follow existing behaviour (`undefined`) if ( (!isNew || [null, undefined].includes(formik.values.image)) && - state.image !== undefined + state.images && + state.images.length > 0 ) { - const imageStr = state.image; + const imageStr = state.images[0]; formik.setFieldValue("image", imageStr ?? undefined); } if (state.details) { @@ -524,20 +525,23 @@ export const PerformerEditPanel: React.FC = ({ const { __typename, - image: _image, + images: _image, tags: _tags, ...ret } = selectedPerformer; const result = await queryScrapePerformer(selectedScraper.id, ret); - if (!result?.data?.scrapePerformer) return; + if (!result?.data?.scrapeSinglePerformer?.length) return; + // assume one result // if this is a new performer, just dump the data if (isNew) { - updatePerformerEditStateFromScraper(result.data.scrapePerformer); + updatePerformerEditStateFromScraper( + result.data.scrapeSinglePerformer[0] + ); setScraper(undefined); } else { - setScrapedPerformer(result.data.scrapePerformer); + setScrapedPerformer(result.data.scrapeSinglePerformer[0]); } } catch (e) { Toast.error(e); @@ -569,12 +573,12 @@ export const PerformerEditPanel: React.FC = ({ } } - async function onScrapeStashBox(performerResult: GQL.ScrapedScenePerformer) { + async function onScrapeStashBox(performerResult: GQL.ScrapedPerformer) { setIsScraperModalOpen(false); - const result: Partial = { + const result: GQL.ScrapedPerformerDataFragment = { ...performerResult, - image: performerResult.images?.[0] ?? undefined, + images: performerResult.images ?? undefined, country: getCountryByISO(performerResult.country), __typename: "ScrapedPerformer", }; diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerMoviesPanel.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerMoviesPanel.tsx new file mode 100644 index 000000000..4176bfb3d --- /dev/null +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerMoviesPanel.tsx @@ -0,0 +1,14 @@ +import React from "react"; +import * as GQL from "src/core/generated-graphql"; +import { MovieList } from "src/components/Movies/MovieList"; +import { performerFilterHook } from "src/core/performers"; + +interface IPerformerDetailsProps { + performer: Partial; +} + +export const PerformerMoviesPanel: React.FC = ({ + performer, +}) => { + return ; +}; diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx index 2a48fca8c..d337fc5c7 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx @@ -97,8 +97,8 @@ function renderScrapedTagsRow( title: string, result: ScrapeResult, onChange: (value: ScrapeResult) => void, - newTags: GQL.ScrapedSceneTag[], - onCreateNew?: (value: GQL.ScrapedSceneTag) => void + newTags: GQL.ScrapedTag[], + onCreateNew?: (value: GQL.ScrapedTag) => void ) { return ( = ( ) ); - const [newTags, setNewTags] = useState( + const [newTags, setNewTags] = useState( props.scraped.tags?.filter((t) => !t.stored_id) ?? [] ); const [image, setImage] = useState>( - new ScrapeResult(props.performer.image, props.scraped.image) + new ScrapeResult( + props.performer.image, + props.scraped.images && props.scraped.images.length > 0 + ? props.scraped.images[0] + : undefined + ) ); const allFields = [ @@ -338,7 +343,7 @@ export const PerformerScrapeDialog: React.FC = ( return <>; } - async function createNewTag(toCreate: GQL.ScrapedSceneTag) { + async function createNewTag(toCreate: GQL.ScrapedTag) { const tagInput: GQL.TagCreateInput = { name: toCreate.name ?? "" }; try { const result = await createTag({ @@ -375,8 +380,9 @@ export const PerformerScrapeDialog: React.FC = ( } function makeNewScrapedItem(): GQL.ScrapedPerformer { + const newImage = image.getNewValue(); return { - name: name.getNewValue(), + name: name.getNewValue() ?? "", aliases: aliases.getNewValue(), birthdate: birthdate.getNewValue(), ethnicity: ethnicity.getNewValue(), @@ -398,7 +404,7 @@ export const PerformerScrapeDialog: React.FC = ( name: "", }; }), - image: image.getNewValue(), + images: newImage ? [newImage] : undefined, details: details.getNewValue(), death_date: deathDate.getNewValue(), hair_color: hairColor.getNewValue(), diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeModal.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeModal.tsx index 823fa4d25..5c02f1f42 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeModal.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeModal.tsx @@ -30,7 +30,7 @@ const PerformerScrapeModal: React.FC = ({ const [query, setQuery] = useState(name ?? ""); const { data, loading } = useScrapePerformerList(scraper.id, query); - const performers = data?.scrapePerformerList ?? []; + const performers = data?.scrapeSinglePerformer ?? []; const onInputChange = debounce((input: string) => { setQuery(input); diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerStashBoxModal.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerStashBoxModal.tsx index 319bb73f6..734c2be18 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerStashBoxModal.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerStashBoxModal.tsx @@ -16,7 +16,7 @@ export interface IStashBox extends GQL.StashBox { interface IProps { instance: IStashBox; onHide: () => void; - onSelectPerformer: (performer: GQL.ScrapedScenePerformer) => void; + onSelectPerformer: (performer: GQL.ScrapedPerformer) => void; name?: string; } const PerformerStashBoxModal: React.FC = ({ @@ -28,17 +28,19 @@ const PerformerStashBoxModal: React.FC = ({ const intl = useIntl(); const inputRef = useRef(null); const [query, setQuery] = useState(name ?? ""); - const { data, loading } = GQL.useQueryStashBoxPerformerQuery({ + const { data, loading } = GQL.useScrapeSinglePerformerQuery({ variables: { - input: { + source: { stash_box_index: instance.index, - q: query, + }, + input: { + query, }, }, skip: query === "", }); - const performers = data?.queryStashBoxPerformer?.[0].results ?? []; + const performers = data?.scrapeSinglePerformer ?? []; const onInputChange = debounce((input: string) => { setQuery(input); diff --git a/ui/v2.5/src/components/ScenePlayer/ScenePlayer.tsx b/ui/v2.5/src/components/ScenePlayer/ScenePlayer.tsx index 77ef81552..1610e253d 100644 --- a/ui/v2.5/src/components/ScenePlayer/ScenePlayer.tsx +++ b/ui/v2.5/src/components/ScenePlayer/ScenePlayer.tsx @@ -3,7 +3,7 @@ import React from "react"; import ReactJWPlayer from "react-jw-player"; import * as GQL from "src/core/generated-graphql"; import { useConfiguration } from "src/core/StashService"; -import { JWUtils } from "src/utils"; +import { JWUtils, ScreenUtils } from "src/utils"; import { ScenePlayerScrubber } from "./ScenePlayerScrubber"; import { Interactive } from "../../utils/interactive"; @@ -294,16 +294,23 @@ export class ScenePlayerImpl extends React.Component< this.playlist = this.makePlaylist(); + // TODO: leverage the floating.mode option after upgrading JWPlayer + const extras: any = {}; + + if (!ScreenUtils.isMobile()) { + extras.floating = { + dismissible: true, + }; + } + const ret = { playlist: this.playlist, image: scene.paths.screenshot, width: "100%", height: "100%", - floating: { - dismissible: true, - }, cast: {}, primary: "html5", + preload: "none", autostart: this.props.autoplay || (this.props.config ? this.props.config.autostartVideo : false) || @@ -314,6 +321,7 @@ export class ScenePlayerImpl extends React.Component< getDurationHook, seekHook, getCurrentTimeHook, + ...extras, }; return ret; @@ -336,12 +344,13 @@ export class ScenePlayerImpl extends React.Component<
this.onComplete()} + className="video-wrapper" /> div:first-child { - /* minus the scrubber height and margin */ - height: calc(100% - #{$scrubberHeight} - 15px); + /* stylelint-disable */ + .jw-video { + // #1764 - jwplayer sets object-fit: fit in the style. Need to override it. + object-fit: contain !important; } -} + /* stylelint-enable */ -/* scrubber is hidden when height < 450px or width < 576, so use entire height for scene player */ -@media (max-height: 449px), (max-width: 575px) { - #jwplayer-container > div:first-child { - height: 100%; + .video-wrapper { + height: 56.25vw; + + @media (min-width: 1200px) { + height: 100%; + } + } + + &.portrait .video-wrapper { + height: 177.78vw; } } @@ -57,9 +52,6 @@ $scrubberHeight: 120px; padding-right: 15px; } } - -$sceneTabWidth: 450px; - @media (min-width: 1200px) { .scene-tabs { flex: 0 0 $sceneTabWidth; @@ -117,6 +109,7 @@ $sceneTabWidth: 450px; } .scrubber-wrapper { + flex-shrink: 0; margin: 5px 0; overflow: hidden; position: relative; diff --git a/ui/v2.5/src/components/Scenes/EditScenesDialog.tsx b/ui/v2.5/src/components/Scenes/EditScenesDialog.tsx index 71e18e641..72237531b 100644 --- a/ui/v2.5/src/components/Scenes/EditScenesDialog.tsx +++ b/ui/v2.5/src/components/Scenes/EditScenesDialog.tsx @@ -33,6 +33,11 @@ export const EditScenesDialog: React.FC = ( ); const [tagIds, setTagIds] = useState(); const [existingTagIds, setExistingTagIds] = useState(); + const [movieMode, setMovieMode] = React.useState( + GQL.BulkUpdateIdMode.Add + ); + const [movieIds, setMovieIds] = useState(); + const [existingMovieIds, setExistingMovieIds] = useState(); const [organized, setOrganized] = useState(); const [updateScenes] = useBulkSceneUpdate(getSceneInput()); @@ -58,6 +63,7 @@ export const EditScenesDialog: React.FC = ( const aggregateStudioId = getStudioId(props.selected); const aggregatePerformerIds = getPerformerIds(props.selected); const aggregateTagIds = getTagIds(props.selected); + const aggregateMovieIds = getMovieIds(props.selected); const sceneInput: GQL.BulkSceneUpdateInput = { ids: props.selected.map((scene) => { @@ -127,6 +133,21 @@ export const EditScenesDialog: React.FC = ( sceneInput.tag_ids = makeBulkUpdateIds(tagIds || [], tagMode); } + // if movieIds non-empty, then we are setting them + if ( + movieMode === GQL.BulkUpdateIdMode.Set && + (!movieIds || movieIds.length === 0) + ) { + // and all scenes have the same ids, + if (aggregateMovieIds.length > 0) { + // then unset the movieIds, otherwise ignore + sceneInput.movie_ids = makeBulkUpdateIds(movieIds || [], movieMode); + } + } else { + // if movieIds non-empty, then we are setting them + sceneInput.movie_ids = makeBulkUpdateIds(movieIds || [], movieMode); + } + if (organized !== undefined) { sceneInput.organized = organized; } @@ -228,12 +249,35 @@ export const EditScenesDialog: React.FC = ( return ret; } + function getMovieIds(state: GQL.SlimSceneDataFragment[]) { + let ret: string[] = []; + let first = true; + + state.forEach((scene: GQL.SlimSceneDataFragment) => { + if (first) { + ret = scene.movies ? scene.movies.map((m) => m.movie.id).sort() : []; + first = false; + } else { + const mIds = scene.movies + ? scene.movies.map((m) => m.movie.id).sort() + : []; + + if (!_.isEqual(ret, mIds)) { + ret = []; + } + } + }); + + return ret; + } + useEffect(() => { const state = props.selected; let updateRating: number | undefined; let updateStudioID: string | undefined; let updatePerformerIds: string[] = []; let updateTagIds: string[] = []; + let updateMovieIds: string[] = []; let updateOrganized: boolean | undefined; let first = true; @@ -244,12 +288,14 @@ export const EditScenesDialog: React.FC = ( .map((p) => p.id) .sort(); const sceneTagIDs = (scene.tags ?? []).map((p) => p.id).sort(); + const sceneMovieIDs = (scene.movies ?? []).map((m) => m.movie.id).sort(); if (first) { updateRating = sceneRating ?? undefined; updateStudioID = sceneStudioID; updatePerformerIds = scenePerformerIDs; updateTagIds = sceneTagIDs; + updateMovieIds = sceneMovieIDs; first = false; updateOrganized = scene.organized; } else { @@ -265,6 +311,9 @@ export const EditScenesDialog: React.FC = ( if (!_.isEqual(sceneTagIDs, updateTagIds)) { updateTagIds = []; } + if (!_.isEqual(sceneMovieIDs, updateMovieIds)) { + updateMovieIds = []; + } if (scene.organized !== updateOrganized) { updateOrganized = undefined; } @@ -275,8 +324,9 @@ export const EditScenesDialog: React.FC = ( setStudioId(updateStudioID); setExistingPerformerIds(updatePerformerIds); setExistingTagIds(updateTagIds); + setExistingMovieIds(updateMovieIds); setOrganized(updateOrganized); - }, [props.selected, performerMode, tagMode]); + }, [props.selected, performerMode, tagMode, movieMode]); useEffect(() => { if (checkboxRef.current) { @@ -285,7 +335,7 @@ export const EditScenesDialog: React.FC = ( }, [organized, checkboxRef]); function renderMultiSelect( - type: "performers" | "tags", + type: "performers" | "tags" | "movies", ids: string[] | undefined ) { let mode = GQL.BulkUpdateIdMode.Add; @@ -299,6 +349,10 @@ export const EditScenesDialog: React.FC = ( mode = tagMode; existingIds = existingTagIds; break; + case "movies": + mode = movieMode; + existingIds = existingMovieIds; + break; } return ( @@ -313,6 +367,9 @@ export const EditScenesDialog: React.FC = ( case "tags": setTagIds(itemIDs); break; + case "movies": + setMovieIds(itemIDs); + break; } }} onSetMode={(newMode) => { @@ -323,6 +380,9 @@ export const EditScenesDialog: React.FC = ( case "tags": setTagMode(newMode); break; + case "movies": + setMovieMode(newMode); + break; } }} ids={ids ?? []} @@ -409,6 +469,13 @@ export const EditScenesDialog: React.FC = ( {renderMultiSelect("tags", tagIds)} + + + + + {renderMultiSelect("movies", movieIds)} + + = ( )); return ( - + diff --git a/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx b/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx index f38749729..fc2cbe54a 100644 --- a/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx +++ b/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx @@ -7,6 +7,7 @@ import { Form, Col, Row, + ButtonGroup, } from "react-bootstrap"; import Mousetrap from "mousetrap"; import * as GQL from "src/core/generated-graphql"; @@ -18,7 +19,7 @@ import { useSceneUpdate, mutateReloadScrapers, useConfiguration, - queryStashBoxScene, + queryScrapeSceneQueryFragment, } from "src/core/StashService"; import { PerformerSelect, @@ -37,6 +38,7 @@ import { Prompt } from "react-router"; import { SceneMovieTable } from "./SceneMovieTable"; import { RatingStars } from "./RatingStars"; import { SceneScrapeDialog } from "./SceneScrapeDialog"; +import { SceneQueryModal } from "./SceneQueryModal"; interface IProps { scene: GQL.SceneDataFragment; @@ -60,8 +62,14 @@ export const SceneEditPanel: React.FC = ({ ); const Scrapers = useListSceneScrapers(); + const [fragmentScrapers, setFragmentScrapers] = useState([]); const [queryableScrapers, setQueryableScrapers] = useState([]); + const [scraper, setScraper] = useState(); + const [ + isScraperQueryModalOpen, + setIsScraperQueryModalOpen, + ] = useState(false); const [scrapedScene, setScrapedScene] = useState(); const [coverImagePreview, setCoverImagePreview] = useState< @@ -181,12 +189,16 @@ export const SceneEditPanel: React.FC = ({ }); useEffect(() => { - const newQueryableScrapers = ( - Scrapers?.data?.listSceneScrapers ?? [] - ).filter((s) => + const toFilter = Scrapers?.data?.listSceneScrapers ?? []; + + const newFragmentScrapers = toFilter.filter((s) => s.scene?.supported_scrapes.includes(GQL.ScrapeType.Fragment) ); + const newQueryableScrapers = toFilter.filter((s) => + s.scene?.supported_scrapes.includes(GQL.ScrapeType.Name) + ); + setFragmentScrapers(newFragmentScrapers); setQueryableScrapers(newQueryableScrapers); }, [Scrapers, stashConfig]); @@ -273,21 +285,18 @@ export const SceneEditPanel: React.FC = ({ ImageUtils.onImageChange(event, onImageLoad); } - async function onScrapeStashBoxClicked(stashBoxIndex: number) { + async function onScrapeClicked(s: GQL.ScraperSourceInput) { setIsLoading(true); try { - const result = await queryStashBoxScene(stashBoxIndex, scene.id); - if (!result.data || !result.data.queryStashBoxScene) { - return; - } - - if (result.data.queryStashBoxScene.length > 0) { - setScrapedScene(result.data.queryStashBoxScene[0]); - } else { + const result = await queryScrapeScene(s, scene.id); + if (!result.data || !result.data.scrapeSingleScene?.length) { Toast.success({ content: "No scenes found", }); + return; } + // assume one returned scene + setScrapedScene(result.data.scrapeSingleScene[0]); } catch (e) { Toast.error(e); } finally { @@ -295,20 +304,29 @@ export const SceneEditPanel: React.FC = ({ } } - async function onScrapeClicked(scraper: GQL.Scraper) { + async function scrapeFromQuery( + s: GQL.ScraperSourceInput, + fragment: GQL.ScrapedSceneDataFragment + ) { setIsLoading(true); try { - const result = await queryScrapeScene( - scraper.id, - getSceneInput(formik.values) - ); - if (!result.data || !result.data.scrapeScene) { + const input: GQL.ScrapedSceneInput = { + date: fragment.date, + details: fragment.details, + remote_site_id: fragment.remote_site_id, + title: fragment.title, + url: fragment.url, + }; + + const result = await queryScrapeSceneQueryFragment(s, input); + if (!result.data || !result.data.scrapeSingleScene?.length) { Toast.success({ content: "No scenes found", }); return; } - setScrapedScene(result.data.scrapeScene); + // assume one returned scene + setScrapedScene(result.data.scrapeSingleScene[0]); } catch (e) { Toast.error(e); } finally { @@ -316,6 +334,11 @@ export const SceneEditPanel: React.FC = ({ } } + function onScrapeQueryClicked(s: GQL.ScraperSourceInput) { + setScraper(s); + setIsScraperQueryModalOpen(true); + } + async function onReloadScrapers() { setIsLoading(true); try { @@ -356,10 +379,79 @@ export const SceneEditPanel: React.FC = ({ ); } + function renderScrapeQueryMenu() { + const stashBoxes = stashConfig.data?.configuration.general.stashBoxes ?? []; + + if (stashBoxes.length === 0 && queryableScrapers.length === 0) return; + + return ( + + + + + + + {stashBoxes.map((s, index) => ( + onScrapeQueryClicked({ stash_box_index: index })} + > + {s.name ?? "Stash-Box"} + + ))} + {queryableScrapers.map((s) => ( + onScrapeQueryClicked({ scraper_id: s.id })} + > + {s.name} + + ))} + onReloadScrapers()}> + + + + + + + + + + ); + } + + function onSceneSelected(s: GQL.ScrapedSceneDataFragment) { + if (!scraper) return; + + if (scraper?.stash_box_index !== undefined) { + // must be stash-box - assume full scene + setScrapedScene(s); + } else { + // must be scraper + scrapeFromQuery(scraper, s); + } + } + + const renderScrapeQueryModal = () => { + if (!isScraperQueryModalOpen || !scraper) return; + + return ( + setScraper(undefined)} + onSelectScene={(s) => { + setIsScraperQueryModalOpen(false); + setScraper(undefined); + onSceneSelected(s); + }} + name={formik.values.title || ""} + /> + ); + }; + function renderScraperMenu() { const stashBoxes = stashConfig.data?.configuration.general.stashBoxes ?? []; - // TODO - change name based on stashbox configuration return ( = ({ {stashBoxes.map((s, index) => ( onScrapeStashBoxClicked(index)} + onClick={() => onScrapeClicked({ stash_box_index: index })} > {s.name ?? "Stash-Box"} ))} - {queryableScrapers.map((s) => ( - onScrapeClicked(s)}> + {fragmentScrapers.map((s) => ( + onScrapeClicked({ scraper_id: s.id })} + > {s.name} ))} @@ -391,44 +486,6 @@ export const SceneEditPanel: React.FC = ({ ); } - function maybeRenderStashboxQueryButton() { - // const stashBoxes = stashConfig.data?.configuration.general.stashBoxes ?? []; - // if (stashBoxes.length === 0) { - // return; - // } - // TODO - hide this button for now, with the view to add it when we get - // the query dialog going - // if (stashBoxes.length === 1) { - // return ( - // - // ); - // } - // // TODO - change name based on stashbox configuration - // return ( - // - // - // - // - // - // {stashBoxes.map((s, index) => ( - // onStashBoxQueryClicked(index)} - // > - // stash-box - // - // ))} - // - // - // ); - } - function urlScrapable(scrapedUrl: string): boolean { return (Scrapers?.data?.listSceneScrapers ?? []).some((s) => (s?.scene?.urls ?? []).some((u) => scrapedUrl.includes(u)) @@ -558,10 +615,11 @@ export const SceneEditPanel: React.FC = ({ message={intl.formatMessage({ id: "dialogs.unsaved_changes" })} /> + {renderScrapeQueryModal()} {maybeRenderScrapeDialog()}
-
+
- - {maybeRenderStashboxQueryButton()} - {renderScraperMenu()} - +
+ + {renderScraperMenu()} + {renderScrapeQueryMenu()} + +
-
+
{renderTextField("title", intl.formatMessage({ id: "title" }))} @@ -625,8 +685,12 @@ export const SceneEditPanel: React.FC = ({ {FormUtils.renderLabel({ title: intl.formatMessage({ id: "galleries" }), + labelProps: { + column: true, + sm: 3, + }, })} - + onSetGalleries(items)} @@ -637,8 +701,12 @@ export const SceneEditPanel: React.FC = ({ {FormUtils.renderLabel({ title: intl.formatMessage({ id: "studios" }), + labelProps: { + column: true, + sm: 3, + }, })} - + formik.setFieldValue( @@ -755,7 +823,7 @@ export const SceneEditPanel: React.FC = ({
-
+
diff --git a/ui/v2.5/src/components/Scenes/SceneDetails/SceneMarkerForm.tsx b/ui/v2.5/src/components/Scenes/SceneDetails/SceneMarkerForm.tsx index e4982c144..6389b8e78 100644 --- a/ui/v2.5/src/components/Scenes/SceneDetails/SceneMarkerForm.tsx +++ b/ui/v2.5/src/components/Scenes/SceneDetails/SceneMarkerForm.tsx @@ -68,30 +68,26 @@ export const SceneMarkerForm: React.FC = ({ .catch((err) => Toast.error(err)); }; const renderTitleField = (fieldProps: FieldProps) => ( -
- - fieldProps.form.setFieldValue("title", query) - } - /> -
+ + fieldProps.form.setFieldValue("title", query) + } + /> ); const renderSecondsField = (fieldProps: FieldProps) => ( -
- fieldProps.form.setFieldValue("seconds", s)} - onReset={() => - fieldProps.form.setFieldValue( - "seconds", - Math.round(JWUtils.getPlayer()?.getPosition() ?? 0) - ) - } - numericValue={Number.parseInt(fieldProps.field.value ?? "0", 10)} - mandatory - /> -
+ fieldProps.form.setFieldValue("seconds", s)} + onReset={() => + fieldProps.form.setFieldValue( + "seconds", + Math.round(JWUtils.getPlayer()?.getPosition() ?? 0) + ) + } + numericValue={Number.parseInt(fieldProps.field.value ?? "0", 10)} + mandatory + /> ); const renderPrimaryTagField = (fieldProps: FieldProps) => ( @@ -100,7 +96,7 @@ export const SceneMarkerForm: React.FC = ({ fieldProps.form.setFieldValue("primaryTagId", tags[0]?.id) } ids={fieldProps.field.value ? [fieldProps.field.value] : []} - noSelectionString="Select or create tag..." + noSelectionString="Select/create tag..." /> ); @@ -114,7 +110,7 @@ export const SceneMarkerForm: React.FC = ({ ) } ids={fieldProps.field.value} - noSelectionString="Select or create tags..." + noSelectionString="Select/create tags..." /> ); @@ -133,28 +129,48 @@ export const SceneMarkerForm: React.FC = ({
- - Scene Marker Title + + Marker Title - {renderTitleField} +
+ {renderTitleField} +
- + Primary Tag -
+
{renderPrimaryTagField}
- - Time - - {renderSecondsField} +
+
+ + Time + +
+ {renderSecondsField} +
+
+
- + Tags -
+
{renderTagsField}
diff --git a/ui/v2.5/src/components/Scenes/SceneDetails/SceneQueryModal.tsx b/ui/v2.5/src/components/Scenes/SceneDetails/SceneQueryModal.tsx new file mode 100644 index 000000000..723c9644f --- /dev/null +++ b/ui/v2.5/src/components/Scenes/SceneDetails/SceneQueryModal.tsx @@ -0,0 +1,229 @@ +import React, { useCallback, useEffect, useRef, useState } from "react"; +import { Badge, Button, Col, Form, InputGroup, Row } from "react-bootstrap"; +import { FormattedMessage, useIntl } from "react-intl"; + +import * as GQL from "src/core/generated-graphql"; +import { + Modal, + LoadingIndicator, + TruncatedText, + Icon, +} from "src/components/Shared"; +import { queryScrapeSceneQuery } from "src/core/StashService"; +import { useToast } from "src/hooks"; + +interface ISceneSearchResultDetailsProps { + scene: GQL.ScrapedSceneDataFragment; +} + +const SceneSearchResultDetails: React.FC = ({ + scene, +}) => { + function renderPerformers() { + if (scene.performers) { + return ( + + + {scene.performers?.map((performer) => ( + + {performer.name} + + ))} + + + ); + } + } + + function renderTags() { + if (scene.tags) { + return ( + + + {scene.tags?.map((tag) => ( + + {tag.name} + + ))} + + + ); + } + } + + function renderImage() { + if (scene.image) { + return ( +
+ +
+ ); + } + } + + return ( +
+ + {renderImage()} +
+

{scene.title}

+
+ {scene.studio?.name} + {scene.studio?.name && scene.date && ` • `} + {scene.date} +
+
+ + + + + + + {renderPerformers()} + {renderTags()} +
+ ); +}; + +export interface ISceneSearchResult { + scene: GQL.ScrapedSceneDataFragment; +} + +export const SceneSearchResult: React.FC = ({ scene }) => { + return ( +
+
+ +
+
+ ); +}; + +interface IProps { + scraper: GQL.ScraperSourceInput; + onHide: () => void; + onSelectScene: (scene: GQL.ScrapedSceneDataFragment) => void; + name?: string; +} +export const SceneQueryModal: React.FC = ({ + scraper, + name, + onHide, + onSelectScene, +}) => { + const CLASSNAME = "SceneScrapeModal"; + const CLASSNAME_LIST = `${CLASSNAME}-list`; + const CLASSNAME_LIST_CONTAINER = `${CLASSNAME_LIST}-container`; + + const intl = useIntl(); + const Toast = useToast(); + + const inputRef = useRef(null); + const [loading, setLoading] = useState(false); + const [scenes, setScenes] = useState(); + const [error, setError] = useState(); + + const doQuery = useCallback( + async (input: string) => { + if (!input) return; + + setLoading(true); + try { + const r = await queryScrapeSceneQuery(scraper, input); + setScenes(r.data.scrapeSingleScene); + } catch (err) { + setError(err); + } finally { + setLoading(false); + } + }, + [scraper] + ); + + useEffect(() => inputRef.current?.focus(), []); + useEffect(() => { + if (error) { + Toast.error(error); + setError(undefined); + } + }, [error, Toast]); + + function renderResults() { + if (!scenes) { + return; + } + + return ( +
+
+ +
+
    + {scenes.map((s, i) => ( + // eslint-disable-next-line jsx-a11y/click-events-have-key-events, jsx-a11y/no-noninteractive-element-interactions, react/no-array-index-key +
  • onSelectScene(s)}> + +
    • + ))} +
+
+ ); + } + + return ( + +
+ + ) => + e.key === "Enter" && doQuery(inputRef.current?.value ?? "") + } + /> + + + + + + {loading ? ( +
+ +
+ ) : ( + renderResults() + )} +
+ + ); +}; diff --git a/ui/v2.5/src/components/Scenes/SceneDetails/SceneScrapeDialog.tsx b/ui/v2.5/src/components/Scenes/SceneDetails/SceneScrapeDialog.tsx index 86989d358..430758f71 100644 --- a/ui/v2.5/src/components/Scenes/SceneDetails/SceneScrapeDialog.tsx +++ b/ui/v2.5/src/components/Scenes/SceneDetails/SceneScrapeDialog.tsx @@ -48,8 +48,8 @@ function renderScrapedStudioRow( title: string, result: ScrapeResult, onChange: (value: ScrapeResult) => void, - newStudio?: GQL.ScrapedSceneStudio, - onCreateNew?: (value: GQL.ScrapedSceneStudio) => void + newStudio?: GQL.ScrapedStudio, + onCreateNew?: (value: GQL.ScrapedStudio) => void ) { return ( , onChange: (value: ScrapeResult) => void, - newPerformers: GQL.ScrapedScenePerformer[], - onCreateNew?: (value: GQL.ScrapedScenePerformer) => void + newPerformers: GQL.ScrapedPerformer[], + onCreateNew?: (value: GQL.ScrapedPerformer) => void ) { + const performersCopy = newPerformers.map((p) => { + const name: string = p.name ?? ""; + return { ...p, name }; + }); + return ( ); @@ -142,9 +147,14 @@ function renderScrapedMoviesRow( title: string, result: ScrapeResult, onChange: (value: ScrapeResult) => void, - newMovies: GQL.ScrapedSceneMovie[], - onCreateNew?: (value: GQL.ScrapedSceneMovie) => void + newMovies: GQL.ScrapedMovie[], + onCreateNew?: (value: GQL.ScrapedMovie) => void ) { + const moviesCopy = newMovies.map((p) => { + const name: string = p.name ?? ""; + return { ...p, name }; + }); + return ( ); @@ -189,8 +199,8 @@ function renderScrapedTagsRow( title: string, result: ScrapeResult, onChange: (value: ScrapeResult) => void, - newTags: GQL.ScrapedSceneTag[], - onCreateNew?: (value: GQL.ScrapedSceneTag) => void + newTags: GQL.ScrapedTag[], + onCreateNew?: (value: GQL.ScrapedTag) => void ) { return ( = ( props.scraped.studio?.stored_id ) ); - const [newStudio, setNewStudio] = useState< - GQL.ScrapedSceneStudio | undefined - >( + const [newStudio, setNewStudio] = useState( props.scraped.studio && !props.scraped.studio.stored_id ? props.scraped.studio : undefined @@ -290,9 +298,9 @@ export const SceneScrapeDialog: React.FC = ( mapStoredIdObjects(props.scraped.performers ?? undefined) ) ); - const [newPerformers, setNewPerformers] = useState< - GQL.ScrapedScenePerformer[] - >(props.scraped.performers?.filter((t) => !t.stored_id) ?? []); + const [newPerformers, setNewPerformers] = useState( + props.scraped.performers?.filter((t) => !t.stored_id) ?? [] + ); const [movies, setMovies] = useState>( new ScrapeResult( @@ -300,7 +308,7 @@ export const SceneScrapeDialog: React.FC = ( mapStoredIdObjects(props.scraped.movies ?? undefined) ) ); - const [newMovies, setNewMovies] = useState( + const [newMovies, setNewMovies] = useState( props.scraped.movies?.filter((t) => !t.stored_id) ?? [] ); @@ -310,7 +318,7 @@ export const SceneScrapeDialog: React.FC = ( mapStoredIdObjects(props.scraped.tags ?? undefined) ) ); - const [newTags, setNewTags] = useState( + const [newTags, setNewTags] = useState( props.scraped.tags?.filter((t) => !t.stored_id) ?? [] ); @@ -339,7 +347,7 @@ export const SceneScrapeDialog: React.FC = ( return <>; } - async function createNewStudio(toCreate: GQL.ScrapedSceneStudio) { + async function createNewStudio(toCreate: GQL.ScrapedStudio) { try { const result = await createStudio({ variables: { @@ -366,7 +374,7 @@ export const SceneScrapeDialog: React.FC = ( } } - async function createNewPerformer(toCreate: GQL.ScrapedScenePerformer) { + async function createNewPerformer(toCreate: GQL.ScrapedPerformer) { const input = makePerformerCreateInput(toCreate); try { @@ -401,7 +409,7 @@ export const SceneScrapeDialog: React.FC = ( } } - async function createNewMovie(toCreate: GQL.ScrapedSceneMovie) { + async function createNewMovie(toCreate: GQL.ScrapedMovie) { let movieInput: GQL.MovieCreateInput = { name: "" }; try { movieInput = Object.assign(movieInput, toCreate); @@ -450,7 +458,7 @@ export const SceneScrapeDialog: React.FC = ( } } - async function createNewTag(toCreate: GQL.ScrapedSceneTag) { + async function createNewTag(toCreate: GQL.ScrapedTag) { const tagInput: GQL.TagCreateInput = { name: toCreate.name ?? "" }; try { const result = await createTag({ diff --git a/ui/v2.5/src/components/Scenes/SceneGenerateDialog.tsx b/ui/v2.5/src/components/Scenes/SceneGenerateDialog.tsx index 776ea7992..4b7ddc547 100644 --- a/ui/v2.5/src/components/Scenes/SceneGenerateDialog.tsx +++ b/ui/v2.5/src/components/Scenes/SceneGenerateDialog.tsx @@ -40,6 +40,8 @@ export const SceneGenerateDialog: React.FC = ( const [previewPreset, setPreviewPreset] = useState( GQL.PreviewPreset.Slow ); + const [markerImagePreviews, setMarkerImagePreviews] = useState(false); + const [markerScreenshots, setMarkerScreenshots] = useState(false); const [previewOptionsOpen, setPreviewOptionsOpen] = useState(false); @@ -67,6 +69,8 @@ export const SceneGenerateDialog: React.FC = ( previews, imagePreviews: previews && imagePreviews, markers, + markerImagePreviews: markers && markerImagePreviews, + markerScreenshots: markers && markerScreenshots, transcodes, overwrite, sceneIDs: props.selectedIds, @@ -276,6 +280,31 @@ export const SceneGenerateDialog: React.FC = ( label={intl.formatMessage({ id: "dialogs.scene_gen.markers" })} onChange={() => setMarkers(!markers)} /> +
+
↳
+ + setMarkerImagePreviews(!markerImagePreviews)} + className="ml-2 flex-grow" + /> + setMarkerScreenshots(!markerScreenshots)} + className="ml-2 flex-grow" + /> + +
.dropdown:not(:last-child) .btn { + border-bottom-right-radius: 0; + border-top-right-radius: 0; + } + + & > .dropdown:not(:first-child) .btn { + border-bottom-left-radius: 0; + border-top-left-radius: 0; + } +} + +.SceneScrapeModal-list { + list-style: none; + max-height: 50vh; + overflow-x: hidden; + overflow-y: auto; + padding-inline-start: 0; + + li { + cursor: pointer; + } +} diff --git a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx index dcd2d153b..1c5b459d4 100644 --- a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx @@ -13,6 +13,7 @@ import StashBoxConfiguration, { IStashBoxInstance, } from "./StashBoxConfiguration"; import StashConfiguration from "./StashConfiguration"; +import { StringListInput } from "../Shared/StringListInput"; interface IExclusionPatternsProps { excludes: string[]; @@ -80,6 +81,9 @@ export const SettingsConfigurationPanel: React.FC = () => { const [generatedPath, setGeneratedPath] = useState( undefined ); + const [metadataPath, setMetadataPath] = useState( + undefined + ); const [cachePath, setCachePath] = useState(undefined); const [calculateMD5, setCalculateMD5] = useState(false); const [videoFileNamingAlgorithm, setVideoFileNamingAlgorithm] = useState< @@ -106,9 +110,13 @@ export const SettingsConfigurationPanel: React.FC = () => { const [maxStreamingTranscodeSize, setMaxStreamingTranscodeSize] = useState< GQL.StreamingResolutionEnum | undefined >(undefined); + const [writeImageThumbnails, setWriteImageThumbnails] = useState(true); const [username, setUsername] = useState(undefined); const [password, setPassword] = useState(undefined); const [maxSessionAge, setMaxSessionAge] = useState(0); + const [trustedProxies, setTrustedProxies] = useState( + undefined + ); const [logFile, setLogFile] = useState(); const [logOut, setLogOut] = useState(true); const [logLevel, setLogLevel] = useState("Info"); @@ -144,6 +152,7 @@ export const SettingsConfigurationPanel: React.FC = () => { })), databasePath, generatedPath, + metadataPath, cachePath, calculateMD5, videoFileNamingAlgorithm: @@ -157,9 +166,11 @@ export const SettingsConfigurationPanel: React.FC = () => { previewPreset: (previewPreset as GQL.PreviewPreset) ?? undefined, maxTranscodeSize, maxStreamingTranscodeSize, + writeImageThumbnails, username, password, maxSessionAge, + trustedProxies, logFile, logOut, logLevel, @@ -189,6 +200,7 @@ export const SettingsConfigurationPanel: React.FC = () => { setStashes(conf.general.stashes ?? []); setDatabasePath(conf.general.databasePath); setGeneratedPath(conf.general.generatedPath); + setMetadataPath(conf.general.metadataPath); setCachePath(conf.general.cachePath); setVideoFileNamingAlgorithm(conf.general.videoFileNamingAlgorithm); setCalculateMD5(conf.general.calculateMD5); @@ -203,9 +215,11 @@ export const SettingsConfigurationPanel: React.FC = () => { setMaxStreamingTranscodeSize( conf.general.maxStreamingTranscodeSize ?? undefined ); + setWriteImageThumbnails(conf.general.writeImageThumbnails); setUsername(conf.general.username); setPassword(conf.general.password); setMaxSessionAge(conf.general.maxSessionAge); + setTrustedProxies(conf.general.trustedProxies ?? undefined); setLogFile(conf.general.logFile ?? undefined); setLogOut(conf.general.logOut); setLogLevel(conf.general.logLevel); @@ -420,6 +434,24 @@ export const SettingsConfigurationPanel: React.FC = () => { + +
+ +
+ ) => + setMetadataPath(e.currentTarget.value) + } + /> + + {intl.formatMessage({ + id: "config.general.metadata_path.description", + })} + + +
@@ -735,11 +767,15 @@ export const SettingsConfigurationPanel: React.FC = () => { setPreviewAudio(!previewAudio)} /> - Includes audio stream when generating previews. + {intl.formatMessage({ + id: "config.general.include_audio_desc", + })} @@ -830,6 +866,30 @@ export const SettingsConfigurationPanel: React.FC = () => { +
+ + +

{intl.formatMessage({ id: "images" })}

+ + + setWriteImageThumbnails(!writeImageThumbnails)} + /> + + {intl.formatMessage({ + id: "config.ui.images.options.write_image_thumbnails.description", + })} + + + + +
+

{intl.formatMessage({ id: "performers" })}

@@ -956,6 +1016,22 @@ export const SettingsConfigurationPanel: React.FC = () => { + +
+ {intl.formatMessage({ id: "config.general.auth.trusted_proxies" })} +
+ setTrustedProxies(value)} + defaultNewValue="" + /> + + {intl.formatMessage({ + id: "config.general.auth.trusted_proxies_desc", + })} + + +

{intl.formatMessage({ id: "config.general.logging" })}

diff --git a/ui/v2.5/src/components/Settings/SettingsInterfacePanel/SettingsInterfacePanel.tsx b/ui/v2.5/src/components/Settings/SettingsInterfacePanel/SettingsInterfacePanel.tsx index e464ae610..311c2ed8b 100644 --- a/ui/v2.5/src/components/Settings/SettingsInterfacePanel/SettingsInterfacePanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsInterfacePanel/SettingsInterfacePanel.tsx @@ -124,6 +124,7 @@ export const SettingsInterfacePanel: React.FC = () => { + diff --git a/ui/v2.5/src/components/Settings/SettingsTasksPanel/GenerateButton.tsx b/ui/v2.5/src/components/Settings/SettingsTasksPanel/GenerateButton.tsx index bf04e36ad..b251d081d 100644 --- a/ui/v2.5/src/components/Settings/SettingsTasksPanel/GenerateButton.tsx +++ b/ui/v2.5/src/components/Settings/SettingsTasksPanel/GenerateButton.tsx @@ -13,6 +13,8 @@ export const GenerateButton: React.FC = () => { const [markers, setMarkers] = useState(true); const [transcodes, setTranscodes] = useState(false); const [imagePreviews, setImagePreviews] = useState(false); + const [markerImagePreviews, setMarkerImagePreviews] = useState(false); + const [markerScreenshots, setMarkerScreenshots] = useState(false); async function onGenerate() { try { @@ -22,6 +24,8 @@ export const GenerateButton: React.FC = () => { previews, imagePreviews: previews && imagePreviews, markers, + markerImagePreviews: markers && markerImagePreviews, + markerScreenshots: markers && markerScreenshots, transcodes, }); Toast.success({ @@ -68,6 +72,31 @@ export const GenerateButton: React.FC = () => { label={intl.formatMessage({ id: "dialogs.scene_gen.markers" })} onChange={() => setMarkers(!markers)} /> +
+
↳
+ + setMarkerImagePreviews(!markerImagePreviews)} + className="ml-2 flex-grow" + /> + setMarkerScreenshots(!markerScreenshots)} + className="ml-2 flex-grow" + /> + +
{ const [scanGeneratePhashes, setScanGeneratePhashes] = useState( false ); + const [scanGenerateThumbnails, setScanGenerateThumbnails] = useState( + false + ); const [cleanDryRun, setCleanDryRun] = useState(false); const [ scanGenerateImagePreviews, @@ -161,6 +164,7 @@ export const SettingsTasksPanel: React.FC = () => { scanGenerateImagePreviews, scanGenerateSprites, scanGeneratePhashes, + scanGenerateThumbnails, }); Toast.success({ content: intl.formatMessage( @@ -403,6 +407,14 @@ export const SettingsTasksPanel: React.FC = () => { })} onChange={() => setScanGeneratePhashes(!scanGeneratePhashes)} /> + setScanGenerateThumbnails(!scanGenerateThumbnails)} + />
); diff --git a/ui/v2.5/src/components/Tags/TagDetails/TagMarkersPanel.tsx b/ui/v2.5/src/components/Tags/TagDetails/TagMarkersPanel.tsx index b520b5905..faf1b4320 100644 --- a/ui/v2.5/src/components/Tags/TagDetails/TagMarkersPanel.tsx +++ b/ui/v2.5/src/components/Tags/TagDetails/TagMarkersPanel.tsx @@ -26,18 +26,21 @@ export const TagMarkersPanel: React.FC = ({ tag }) => { ) { // add the tag if not present if ( - !tagCriterion.value.find((p) => { + !tagCriterion.value.items.find((p) => { return p.id === tag.id; }) ) { - tagCriterion.value.push(tagValue); + tagCriterion.value.items.push(tagValue); } tagCriterion.modifier = GQL.CriterionModifier.IncludesAll; } else { // overwrite tagCriterion = new TagsCriterion(TagsCriterionOption); - tagCriterion.value = [tagValue]; + tagCriterion.value = { + items: [tagValue], + depth: 0, + }; filter.criteria.push(tagCriterion); } diff --git a/ui/v2.5/src/components/Tags/TagList.tsx b/ui/v2.5/src/components/Tags/TagList.tsx index d96320e85..88e804a82 100644 --- a/ui/v2.5/src/components/Tags/TagList.tsx +++ b/ui/v2.5/src/components/Tags/TagList.tsx @@ -24,6 +24,7 @@ import { NavUtils } from "src/utils"; import { Icon, Modal, DeleteEntityDialog } from "src/components/Shared"; import { TagCard } from "./TagCard"; import { ExportDialog } from "../Shared/ExportDialog"; +import { tagRelationHook } from "../../core/tags"; interface ITagList { filterHook?: (filter: ListFilterModel) => ListFilterModel; @@ -138,6 +139,15 @@ export const TagList: React.FC = ({ filterHook }) => { singularEntity={intl.formatMessage({ id: "tag" })} pluralEntity={intl.formatMessage({ id: "tags" })} destroyMutation={useTagsDestroy} + onDeleted={() => { + selectedTags.forEach((t) => + tagRelationHook( + t, + { parents: t.parents ?? [], children: t.children ?? [] }, + { parents: [], children: [] } + ) + ); + }} /> ); @@ -175,7 +185,15 @@ export const TagList: React.FC = ({ filterHook }) => { async function onDelete() { try { + const oldRelations = { + parents: deletingTag?.parents ?? [], + children: deletingTag?.children ?? [], + }; await deleteTag(); + tagRelationHook(deletingTag as GQL.TagDataFragment, oldRelations, { + parents: [], + children: [], + }); Toast.success({ content: intl.formatMessage( { id: "toast.delete_past_tense" }, diff --git a/ui/v2.5/src/components/Wall/WallItem.tsx b/ui/v2.5/src/components/Wall/WallItem.tsx index 0764eab3c..a824ab1f6 100644 --- a/ui/v2.5/src/components/Wall/WallItem.tsx +++ b/ui/v2.5/src/components/Wall/WallItem.tsx @@ -114,6 +114,7 @@ export const WallItem: React.FC = (props: IWallItemProps) => { ? { video: props.sceneMarker.stream, animation: props.sceneMarker.preview, + image: props.sceneMarker.screenshot, } : props.scene ? { diff --git a/ui/v2.5/src/core/StashService.ts b/ui/v2.5/src/core/StashService.ts index 1762a6ff9..2a2e4abd4 100644 --- a/ui/v2.5/src/core/StashService.ts +++ b/ui/v2.5/src/core/StashService.ts @@ -257,17 +257,17 @@ export const useSceneMarkerDestroy = () => export const useListPerformerScrapers = () => GQL.useListPerformerScrapersQuery(); export const useScrapePerformerList = (scraperId: string, q: string) => - GQL.useScrapePerformerListQuery({ - variables: { scraper_id: scraperId, query: q }, + GQL.useScrapeSinglePerformerQuery({ + variables: { + source: { + scraper_id: scraperId, + }, + input: { + query: q, + }, + }, skip: q === "", }); -export const useScrapePerformer = ( - scraperId: string, - scrapedPerformer: GQL.ScrapedPerformerInput -) => - GQL.useScrapePerformerQuery({ - variables: { scraper_id: scraperId, scraped_performer: scrapedPerformer }, - }); export const useListSceneScrapers = () => GQL.useListSceneScrapersQuery(); @@ -814,11 +814,15 @@ export const queryScrapePerformer = ( scraperId: string, scrapedPerformer: GQL.ScrapedPerformerInput ) => - client.query({ - query: GQL.ScrapePerformerDocument, + client.query({ + query: GQL.ScrapeSinglePerformerDocument, variables: { - scraper_id: scraperId, - scraped_performer: scrapedPerformer, + source: { + scraper_id: scraperId, + }, + input: { + performer_input: scrapedPerformer, + }, }, fetchPolicy: "network-only", }); @@ -832,6 +836,21 @@ export const queryScrapePerformerURL = (url: string) => fetchPolicy: "network-only", }); +export const queryScrapeSceneQuery = ( + source: GQL.ScraperSourceInput, + q: string +) => + client.query({ + query: GQL.ScrapeSingleSceneDocument, + variables: { + source, + input: { + query: q, + }, + }, + fetchPolicy: "network-only", + }); + export const queryScrapeSceneURL = (url: string) => client.query({ query: GQL.ScrapeSceneUrlDocument, @@ -860,52 +879,59 @@ export const queryScrapeMovieURL = (url: string) => }); export const queryScrapeScene = ( - scraperId: string, - scene: GQL.SceneUpdateInput + source: GQL.ScraperSourceInput, + sceneId: string ) => - client.query({ - query: GQL.ScrapeSceneDocument, + client.query({ + query: GQL.ScrapeSingleSceneDocument, variables: { - scraper_id: scraperId, - scene, + source, + input: { + scene_id: sceneId, + }, }, fetchPolicy: "network-only", }); export const queryStashBoxScene = (stashBoxIndex: number, sceneID: string) => - client.query({ - query: GQL.QueryStashBoxSceneDocument, + client.query({ + query: GQL.ScrapeSingleSceneDocument, variables: { - input: { + source: { stash_box_index: stashBoxIndex, - scene_ids: [sceneID], + }, + input: { + scene_id: sceneID, }, }, + fetchPolicy: "network-only", }); -export const queryStashBoxPerformer = ( - stashBoxIndex: number, - performerID: string +export const queryScrapeSceneQueryFragment = ( + source: GQL.ScraperSourceInput, + input: GQL.ScrapedSceneInput ) => - client.query({ - query: GQL.QueryStashBoxPerformerDocument, + client.query({ + query: GQL.ScrapeSingleSceneDocument, variables: { + source, input: { - stash_box_index: stashBoxIndex, - performer_ids: [performerID], + scene_input: input, }, }, + fetchPolicy: "network-only", }); -export const queryScrapeGallery = ( - scraperId: string, - gallery: GQL.GalleryUpdateInput -) => - client.query({ - query: GQL.ScrapeGalleryDocument, +export const queryScrapeGallery = (scraperId: string, galleryId: string) => + client.query({ + query: GQL.ScrapeSingleGalleryDocument, variables: { - scraper_id: scraperId, - gallery, + source: { + scraper_id: scraperId, + }, + input: { + gallery_id: galleryId, + }, }, fetchPolicy: "network-only", }); @@ -1017,11 +1043,9 @@ export const queryParseSceneFilenames = ( fetchPolicy: "network-only", }); -export const makePerformerCreateInput = ( - toCreate: GQL.ScrapedScenePerformer -) => { +export const makePerformerCreateInput = (toCreate: GQL.ScrapedPerformer) => { const input: GQL.PerformerCreateInput = { - name: toCreate.name, + name: toCreate.name ?? "", url: toCreate.url, gender: stringToGender(toCreate.gender), birthdate: toCreate.birthdate, @@ -1051,37 +1075,47 @@ export const makePerformerCreateInput = ( }; export const stashBoxSceneQuery = (searchVal: string, stashBoxIndex: number) => - client?.query< - GQL.QueryStashBoxSceneQuery, - GQL.QueryStashBoxSceneQueryVariables - >({ - query: GQL.QueryStashBoxSceneDocument, - variables: { input: { q: searchVal, stash_box_index: stashBoxIndex } }, + client.query({ + query: GQL.ScrapeSingleSceneDocument, + variables: { + source: { + stash_box_index: stashBoxIndex, + }, + input: { + query: searchVal, + }, + }, }); export const stashBoxPerformerQuery = ( searchVal: string, stashBoxIndex: number ) => - client?.query< - GQL.QueryStashBoxPerformerQuery, - GQL.QueryStashBoxPerformerQueryVariables - >({ - query: GQL.QueryStashBoxPerformerDocument, - variables: { input: { q: searchVal, stash_box_index: stashBoxIndex } }, + client.query({ + query: GQL.ScrapeSinglePerformerDocument, + variables: { + source: { + stash_box_index: stashBoxIndex, + }, + input: { + query: searchVal, + }, + }, }); export const stashBoxSceneBatchQuery = ( sceneIds: string[], stashBoxIndex: number ) => - client?.query< - GQL.QueryStashBoxSceneQuery, - GQL.QueryStashBoxSceneQueryVariables - >({ - query: GQL.QueryStashBoxSceneDocument, + client.query({ + query: GQL.ScrapeMultiScenesDocument, variables: { - input: { scene_ids: sceneIds, stash_box_index: stashBoxIndex }, + source: { + stash_box_index: stashBoxIndex, + }, + input: { + scene_ids: sceneIds, + }, }, }); @@ -1089,12 +1123,14 @@ export const stashBoxPerformerBatchQuery = ( performerIds: string[], stashBoxIndex: number ) => - client?.query< - GQL.QueryStashBoxPerformerQuery, - GQL.QueryStashBoxPerformerQueryVariables - >({ - query: GQL.QueryStashBoxPerformerDocument, + client.query({ + query: GQL.ScrapeMultiPerformersDocument, variables: { - input: { performer_ids: performerIds, stash_box_index: stashBoxIndex }, + source: { + stash_box_index: stashBoxIndex, + }, + input: { + performer_ids: performerIds, + }, }, }); diff --git a/ui/v2.5/src/core/createClient.ts b/ui/v2.5/src/core/createClient.ts index ce976be77..134417bd8 100644 --- a/ui/v2.5/src/core/createClient.ts +++ b/ui/v2.5/src/core/createClient.ts @@ -68,10 +68,27 @@ const typePolicies: TypePolicies = { }, }, }, + + Tag: { + fields: { + parents: { + merge: false, + }, + children: { + merge: false, + }, + }, + }, +}; + +export const getBaseURL = () => { + const baseURL = window.STASH_BASE_URL; + if (baseURL === "%BASE_URL%") return "/"; + return baseURL; }; export const getPlatformURL = (ws?: boolean) => { - const platformUrl = new URL(window.location.origin); + const platformUrl = new URL(window.location.origin + getBaseURL()); if (!process.env.NODE_ENV || process.env.NODE_ENV === "development") { platformUrl.port = process.env.REACT_APP_PLATFORM_PORT ?? "9999"; @@ -96,8 +113,8 @@ export const createClient = () => { wsPlatformUrl.protocol = "wss:"; } - const url = `${platformUrl.toString().slice(0, -1)}/graphql`; - const wsUrl = `${wsPlatformUrl.toString().slice(0, -1)}/graphql`; + const url = `${platformUrl.toString()}graphql`; + const wsUrl = `${wsPlatformUrl.toString()}graphql`; const httpLink = createUploadLink({ uri: url, @@ -114,7 +131,10 @@ export const createClient = () => { // handle unauthorized error by redirecting to the login page if (networkError && (networkError as ServerError).statusCode === 401) { // redirect to login page - const newURL = new URL("/login", window.location.toString()); + const newURL = new URL( + `${window.STASH_BASE_URL}login`, + window.location.toString() + ); newURL.searchParams.append("returnURL", window.location.href); window.location.href = newURL.toString(); } diff --git a/ui/v2.5/src/core/tags.ts b/ui/v2.5/src/core/tags.ts index 0c0afed61..d7365b8ac 100644 --- a/ui/v2.5/src/core/tags.ts +++ b/ui/v2.5/src/core/tags.ts @@ -1,4 +1,6 @@ +import { gql } from "@apollo/client"; import * as GQL from "src/core/generated-graphql"; +import { getClient } from "src/core/StashService"; import { TagsCriterion, TagsCriterionOption, @@ -20,21 +22,83 @@ export const tagFilterHook = (tag: GQL.TagDataFragment) => { ) { // add the tag if not present if ( - !tagCriterion.value.find((p) => { + !tagCriterion.value.items.find((p) => { return p.id === tag.id; }) ) { - tagCriterion.value.push(tagValue); + tagCriterion.value.items.push(tagValue); } tagCriterion.modifier = GQL.CriterionModifier.IncludesAll; } else { // overwrite tagCriterion = new TagsCriterion(TagsCriterionOption); - tagCriterion.value = [tagValue]; + tagCriterion.value = { + items: [tagValue], + depth: 0, + }; filter.criteria.push(tagCriterion); } return filter; }; }; + +interface ITagRelationTuple { + parents: GQL.SlimTagDataFragment[]; + children: GQL.SlimTagDataFragment[]; +} + +export const tagRelationHook = ( + tag: GQL.SlimTagDataFragment | GQL.TagDataFragment, + old: ITagRelationTuple, + updated: ITagRelationTuple +) => { + const { cache } = getClient(); + + const tagRef = cache.writeFragment({ + data: tag, + fragment: gql` + fragment Tag on Tag { + id + } + `, + }); + + function updater( + property: "parents" | "children", + oldTags: GQL.SlimTagDataFragment[], + updatedTags: GQL.SlimTagDataFragment[] + ) { + oldTags.forEach((o) => { + if (!updatedTags.some((u) => u.id === o.id)) { + cache.modify({ + id: cache.identify(o), + fields: { + [property](value, { readField }) { + return value.filter( + (t: GQL.SlimTagDataFragment) => readField("id", t) !== tag.id + ); + }, + }, + }); + } + }); + + updatedTags.forEach((u) => { + if (!oldTags.some((o) => o.id === u.id)) { + cache.modify({ + id: cache.identify(u), + fields: { + [property](value) { + return [...value, tagRef]; + }, + }, + }); + } + }); + } + + updater("children", old.parents, updated.parents); + updater("parents", old.children, updated.children); +}; diff --git a/ui/v2.5/src/docs/en/Deduplication.md b/ui/v2.5/src/docs/en/Deduplication.md index e44d535e6..24c0fb391 100644 --- a/ui/v2.5/src/docs/en/Deduplication.md +++ b/ui/v2.5/src/docs/en/Deduplication.md @@ -1,6 +1,6 @@ # Dupe Checker -[The dupe checker](/settings?tab=duplicates) searches your collection for scenes that are perceptually similar. This means that the files don't need to be identical, and will be identified even with different bitrates, resolutions, and intros/outros. +[The dupe checker](/sceneDuplicateChecker) searches your collection for scenes that are perceptually similar. This means that the files don't need to be identical, and will be identified even with different bitrates, resolutions, and intros/outros. To achieve this stash needs to generate what's called a phash, or perceptual hash. Similar to sprite generation stash will generate a set of 25 images from fixed points in the scene. These images will be stitched together, and then hashed using the phash algorithm. The phash can then be used to find scenes that are the same or similar to others in the database. Phash generation can be run during scan, or as a separate task. Note that generation can take a while due to the work involved with extracting screenshots. diff --git a/ui/v2.5/src/docs/en/Scraping.md b/ui/v2.5/src/docs/en/Scraping.md index 9dd8374a9..9f072a3ce 100644 --- a/ui/v2.5/src/docs/en/Scraping.md +++ b/ui/v2.5/src/docs/en/Scraping.md @@ -40,6 +40,10 @@ performerByFragment: performerByURL: +sceneByName: + +sceneByQueryFragment: + sceneByFragment: sceneByURL: @@ -63,6 +67,7 @@ The scraping types and their required fields are outlined in the following table |-----------|------------------------| | Scraper in `Scrape...` dropdown button in Performer Edit page | Valid `performerByName` and `performerByFragment` configurations. | | Scrape performer from URL | Valid `performerByURL` configuration with matching URL. | +| Scraper in query dropdown button in Scene Edit page | Valid `sceneByName` and `sceneByQueryFragment` configurations. | | Scraper in `Scrape...` dropdown button in Scene Edit page | Valid `sceneByFragment` configuration. | | Scrape scene from URL | Valid `sceneByURL` configuration with matching URL. | | Scrape movie from URL | Valid `movieByURL` configuration with matching URL. | @@ -97,7 +102,8 @@ The script is sent input and expects output based on the scraping type, as detai | `performerByName` | `{"name": ""}` | Array of JSON-encoded performer fragments (including at least `name`) | | `performerByFragment` | JSON-encoded performer fragment | JSON-encoded performer fragment | | `performerByURL` | `{"url": ""}` | JSON-encoded performer fragment | -| `sceneByFragment` | JSON-encoded scene fragment | JSON-encoded scene fragment | +| `sceneByName` | `{"name": ""}` | Array of JSON-encoded scene fragments | +| `sceneByQueryFragment`, `sceneByFragment` | JSON-encoded scene fragment | JSON-encoded scene fragment | | `sceneByURL` | `{"url": ""}` | JSON-encoded scene fragment | | `movieByURL` | `{"url": ""}` | JSON-encoded movie fragment | | `galleryByFragment` | JSON-encoded gallery fragment | JSON-encoded gallery fragment | @@ -217,9 +223,9 @@ xPathScrapers: # ... performer scraper details ... ``` -### scrapeXPath and scrapeJson use with `sceneByFragment` +### scrapeXPath and scrapeJson use with `sceneByFragment` and `sceneByQueryFragment` -For `sceneByFragment`, the `queryURL` field must also be present. This field is used to build a query URL for scenes. For `sceneByFragment`, the `queryURL` field supports the following placeholder fields: +For `sceneByFragment` and `sceneByQueryFragment`, the `queryURL` field must also be present. This field is used to build a query URL for scenes. For `sceneByFragment`, the `queryURL` field supports the following placeholder fields: * `{checksum}` - the MD5 checksum of the scene * `{oshash}` - the oshash of the scene * `{filename}` - the base filename of the scene diff --git a/ui/v2.5/src/globals.d.ts b/ui/v2.5/src/globals.d.ts index 08dbb94ad..20db3882a 100644 --- a/ui/v2.5/src/globals.d.ts +++ b/ui/v2.5/src/globals.d.ts @@ -1,3 +1,5 @@ +// eslint-disable-next-line no-var +declare var STASH_BASE_URL: string; declare module "*.md"; declare module "string.prototype.replaceall"; declare module "mousetrap-pause"; diff --git a/ui/v2.5/src/hooks/Lightbox/Lightbox.tsx b/ui/v2.5/src/hooks/Lightbox/Lightbox.tsx index 4d2c7d81a..8b5fc8932 100644 --- a/ui/v2.5/src/hooks/Lightbox/Lightbox.tsx +++ b/ui/v2.5/src/hooks/Lightbox/Lightbox.tsx @@ -3,11 +3,11 @@ import * as GQL from "src/core/generated-graphql"; import { Button, Col, - FormControl, InputGroup, - FormLabel, - OverlayTrigger, + Overlay, Popover, + Form, + Row, } from "react-bootstrap"; import cx from "classnames"; import Mousetrap from "mousetrap"; @@ -16,14 +16,16 @@ import debounce from "lodash/debounce"; import { Icon, LoadingIndicator } from "src/components/Shared"; import { useInterval, usePageVisibility } from "src/hooks"; import { useConfiguration } from "src/core/StashService"; +import { FormattedMessage, useIntl } from "react-intl"; +import { DisplayMode, LightboxImage, ScrollMode } from "./LightboxImage"; const CLASSNAME = "Lightbox"; const CLASSNAME_HEADER = `${CLASSNAME}-header`; const CLASSNAME_LEFT_SPACER = `${CLASSNAME_HEADER}-left-spacer`; const CLASSNAME_INDICATOR = `${CLASSNAME_HEADER}-indicator`; -const CLASSNAME_DELAY = `${CLASSNAME_HEADER}-delay`; -const CLASSNAME_DELAY_ICON = `${CLASSNAME_DELAY}-icon`; -const CLASSNAME_DELAY_INLINE = `${CLASSNAME_DELAY}-inline`; +const CLASSNAME_OPTIONS = `${CLASSNAME_HEADER}-options`; +const CLASSNAME_OPTIONS_ICON = `${CLASSNAME_OPTIONS}-icon`; +const CLASSNAME_OPTIONS_INLINE = `${CLASSNAME_OPTIONS}-inline`; const CLASSNAME_RIGHT = `${CLASSNAME_HEADER}-right`; const CLASSNAME_DISPLAY = `${CLASSNAME}-display`; const CLASSNAME_CAROUSEL = `${CLASSNAME}-carousel`; @@ -31,7 +33,6 @@ const CLASSNAME_INSTANT = `${CLASSNAME_CAROUSEL}-instant`; const CLASSNAME_IMAGE = `${CLASSNAME_CAROUSEL}-image`; const CLASSNAME_NAVBUTTON = `${CLASSNAME}-navbutton`; const CLASSNAME_NAV = `${CLASSNAME}-nav`; -const CLASSNAME_NAVZONE = `${CLASSNAME}-navzone`; const CLASSNAME_NAVIMAGE = `${CLASSNAME_NAV}-image`; const CLASSNAME_NAVSELECTED = `${CLASSNAME_NAV}-selected`; @@ -48,7 +49,7 @@ interface IProps { showNavigation: boolean; slideshowEnabled?: boolean; pageHeader?: string; - pageCallback?: (direction: number) => boolean; + pageCallback?: (direction: number) => void; hide: () => void; } @@ -63,16 +64,35 @@ export const LightboxComponent: React.FC = ({ pageCallback, hide, }) => { - const index = useRef(null); + const [index, setIndex] = useState(null); + const oldIndex = useRef(null); const [instantTransition, setInstantTransition] = useState(false); - const [isSwitchingPage, setIsSwitchingPage] = useState(false); + const [isSwitchingPage, setIsSwitchingPage] = useState(true); const [isFullscreen, setFullscreen] = useState(false); + const [showOptions, setShowOptions] = useState(false); + + const oldImages = useRef([]); + + const [displayMode, setDisplayMode] = useState(DisplayMode.FIT_XY); + const oldDisplayMode = useRef(displayMode); + + const [scaleUp, setScaleUp] = useState(false); + const [scrollMode, setScrollMode] = useState(ScrollMode.ZOOM); + const [resetZoomOnNav, setResetZoomOnNav] = useState(true); + const [zoom, setZoom] = useState(1); + const [resetPosition, setResetPosition] = useState(false); + const containerRef = useRef(null); + const overlayTarget = useRef(null); const carouselRef = useRef(null); const indicatorRef = useRef(null); const navRef = useRef(null); const clearIntervalCallback = useRef<() => void>(); const resetIntervalCallback = useRef<() => void>(); + + const allowNavigation = images.length > 1 || pageCallback; + + const intl = useIntl(); const config = useConfiguration(); const userSelectedSlideshowDelayOrDefault = @@ -94,9 +114,12 @@ export const LightboxComponent: React.FC = ({ ); useEffect(() => { - setIsSwitchingPage(false); - if (index.current === -1) index.current = images.length - 1; - }, [images]); + if (images !== oldImages.current && isSwitchingPage) { + oldImages.current = images; + if (index === -1) setIndex(images.length - 1); + setIsSwitchingPage(false); + } + }, [isSwitchingPage, images, index]); const disableInstantTransition = debounce( () => setInstantTransition(false), @@ -108,35 +131,56 @@ export const LightboxComponent: React.FC = ({ disableInstantTransition(); }, [disableInstantTransition]); - const setIndex = useCallback( - (i: number) => { - if (images.length < 2) return; + useEffect(() => { + if (images.length < 2) return; + if (index === oldIndex.current) return; + if (index === null) return; - index.current = i; - if (carouselRef.current) carouselRef.current.style.left = `${i * -100}vw`; - if (indicatorRef.current) - indicatorRef.current.innerHTML = `${i + 1} / ${images.length}`; - if (navRef.current) { - const currentThumb = navRef.current.children[i + 1]; - if (currentThumb instanceof HTMLImageElement) { - const offset = - -1 * - (currentThumb.offsetLeft - - document.documentElement.clientWidth / 2); - navRef.current.style.left = `${offset}px`; + // reset zoom status + // setResetZoom((r) => !r); + // setZoomed(false); + if (resetZoomOnNav) { + setZoom(1); + } + setResetPosition((r) => !r); - const previouslySelected = navRef.current.getElementsByClassName( - CLASSNAME_NAVSELECTED - )?.[0]; - if (previouslySelected) - previouslySelected.className = CLASSNAME_NAVIMAGE; + if (carouselRef.current) + carouselRef.current.style.left = `${index * -100}vw`; + if (indicatorRef.current) + indicatorRef.current.innerHTML = `${index + 1} / ${images.length}`; + if (navRef.current) { + const currentThumb = navRef.current.children[index + 1]; + if (currentThumb instanceof HTMLImageElement) { + const offset = + -1 * + (currentThumb.offsetLeft - document.documentElement.clientWidth / 2); + navRef.current.style.left = `${offset}px`; - currentThumb.className = `${CLASSNAME_NAVIMAGE} ${CLASSNAME_NAVSELECTED}`; - } + const previouslySelected = navRef.current.getElementsByClassName( + CLASSNAME_NAVSELECTED + )?.[0]; + if (previouslySelected) + previouslySelected.className = CLASSNAME_NAVIMAGE; + + currentThumb.className = `${CLASSNAME_NAVIMAGE} ${CLASSNAME_NAVSELECTED}`; } - }, - [images] - ); + } + + oldIndex.current = index; + }, [index, images.length, resetZoomOnNav]); + + useEffect(() => { + if (displayMode !== oldDisplayMode.current) { + // reset zoom status + // setResetZoom((r) => !r); + // setZoomed(false); + if (resetZoomOnNav) { + setZoom(1); + } + setResetPosition((r) => !r); + } + oldDisplayMode.current = displayMode; + }, [displayMode, resetZoomOnNav]); const selectIndex = (e: React.MouseEvent, i: number) => { setIndex(i); @@ -145,12 +189,12 @@ export const LightboxComponent: React.FC = ({ useEffect(() => { if (isVisible) { - if (index.current === null) setIndex(initialIndex); + if (index === null) setIndex(initialIndex); document.body.style.overflow = "hidden"; // eslint-disable-next-line @typescript-eslint/no-explicit-any (Mousetrap as any).pause(); } - }, [initialIndex, isVisible, setIndex]); + }, [initialIndex, isVisible, setIndex, index]); const toggleSlideshow = useCallback(() => { if (slideshowInterval) { @@ -185,54 +229,55 @@ export const LightboxComponent: React.FC = ({ const handleClose = (e: React.MouseEvent) => { const { className } = e.target as Element; - if (className === CLASSNAME_IMAGE) close(); + if (className && className.includes && className.includes(CLASSNAME_IMAGE)) + close(); }; const handleLeft = useCallback( (isUserAction = true) => { - if (isSwitchingPage || index.current === -1) return; + if (isSwitchingPage || index === -1) return; - if (index.current === 0) { + if (index === 0) { + // go to next page, or loop back if no callback is set if (pageCallback) { - setIsSwitchingPage(true); + pageCallback(-1); setIndex(-1); - // Check if calling page wants to swap page - const repage = pageCallback(-1); - if (!repage) { - setIsSwitchingPage(false); - setIndex(0); - } + setIsSwitchingPage(true); } else setIndex(images.length - 1); - } else setIndex((index.current ?? 0) - 1); + } else setIndex((index ?? 0) - 1); if (isUserAction && resetIntervalCallback.current) { resetIntervalCallback.current(); } }, - [images, setIndex, pageCallback, isSwitchingPage, resetIntervalCallback] + [images, pageCallback, isSwitchingPage, resetIntervalCallback, index] ); const handleRight = useCallback( (isUserAction = true) => { if (isSwitchingPage) return; - if (index.current === images.length - 1) { + if (index === images.length - 1) { + // go to preview page, or loop back if no callback is set if (pageCallback) { + pageCallback(1); setIsSwitchingPage(true); setIndex(0); - const repage = pageCallback?.(1); - if (!repage) { - setIsSwitchingPage(false); - setIndex(images.length - 1); - } } else setIndex(0); - } else setIndex((index.current ?? 0) + 1); + } else setIndex((index ?? 0) + 1); if (isUserAction && resetIntervalCallback.current) { resetIntervalCallback.current(); } }, - [images, setIndex, pageCallback, isSwitchingPage, resetIntervalCallback] + [ + images, + setIndex, + pageCallback, + isSwitchingPage, + resetIntervalCallback, + index, + ] ); const handleKey = useCallback( @@ -252,51 +297,6 @@ export const LightboxComponent: React.FC = ({ setFullscreen(document.fullscreenElement !== null); }; - const handleTouchStart = (ev: React.TouchEvent) => { - setInstantTransition(true); - - const el = ev.currentTarget; - if (ev.touches.length !== 1) return; - - const startX = ev.touches[0].clientX; - let position = 0; - - const resetPosition = () => { - if (carouselRef.current) - carouselRef.current.style.left = `${(index.current ?? 0) * -100}vw`; - }; - const handleMove = (e: TouchEvent) => { - position = e.touches[0].clientX; - if (carouselRef.current) - carouselRef.current.style.left = `calc(${ - (index.current ?? 0) * -100 - }vw + ${e.touches[0].clientX - startX}px)`; - }; - const handleEnd = () => { - const diff = position - startX; - if (diff <= -50) handleRight(); - else if (diff >= 50) handleLeft(); - else resetPosition(); - // eslint-disable-next-line @typescript-eslint/no-use-before-define - cleanup(); - }; - const handleCancel = () => { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - cleanup(); - resetPosition(); - }; - const cleanup = () => { - el.removeEventListener("touchmove", handleMove); - el.removeEventListener("touchend", handleEnd); - el.removeEventListener("touchcancel", handleCancel); - setInstantTransition(false); - }; - - el.addEventListener("touchmove", handleMove); - el.addEventListener("touchend", handleEnd); - el.addEventListener("touchcancel", handleCancel); - }; - const [clearCallback, resetCallback] = useInterval( () => { handleRight(false); @@ -332,7 +332,7 @@ export const LightboxComponent: React.FC = ({ src={image.paths.thumbnail ?? ""} alt="" className={cx(CLASSNAME_NAVIMAGE, { - [CLASSNAME_NAVSELECTED]: i === index.current, + [CLASSNAME_NAVSELECTED]: i === index, })} onClick={(e: React.MouseEvent) => selectIndex(e, i)} role="presentation" @@ -359,36 +359,142 @@ export const LightboxComponent: React.FC = ({ } }; - const currentIndex = index.current === null ? initialIndex : index.current; + const currentIndex = index === null ? initialIndex : index; - const DelayForm: React.FC<{}> = () => ( + const OptionsForm: React.FC<{}> = () => ( <> - - Delay (Sec) - - - - + {slideshowEnabled ? ( + + + + + + + + + + + ) : undefined} + + + + + + + + + setDisplayMode(e.target.value as DisplayMode)} + value={displayMode} + className="btn-secondary mx-1 mb-1" + > + + + + + + + + + + setScaleUp(v.currentTarget.checked)} + /> + + + + {intl.formatMessage({ + id: "dialogs.lightbox.scale_up.description", + })} + + + + + + setResetZoomOnNav(v.currentTarget.checked)} + /> + + + + + + + + + + + + setScrollMode(e.target.value as ScrollMode)} + value={scrollMode} + className="btn-secondary mx-1 mb-1" + > + + + + + + + {intl.formatMessage({ + id: "dialogs.lightbox.scroll_mode.description", + })} + + ); - const delayPopover = ( - - Set slideshow delay + const optionsPopover = ( + <> + + {intl.formatMessage({ + id: "dialogs.lightbox.options", + })} + - - - + - + ); const element = isVisible ? ( @@ -396,7 +502,7 @@ export const LightboxComponent: React.FC = ({ className={CLASSNAME} role="presentation" ref={containerRef} - onMouseDown={handleClose} + onClick={handleClose} > {images.length > 0 && !isLoading && !isSwitchingPage ? ( <> @@ -409,34 +515,59 @@ export const LightboxComponent: React.FC = ({
- {slideshowEnabled && ( - <> -
-
- - - -
- - - -
+
+
- + setShowOptions(false)} + > + {({ placement, arrowProps, show: _show, ...props }) => ( +
+ {optionsPopover} +
+ )} + +
+ + + +
+ {slideshowEnabled && ( + + )} + {zoom !== 1 && ( + )} {document.fullscreenEnabled && (
-
- {images.length > 1 && ( +
+ {allowNavigation && (