Backup_Repos/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2026-05-09 05:05:29 +02:00
Code Issues Projects Releases Wiki Activity

Merge branch 'stashapp:develop' into develop

Browse source
This commit is contained in:
bob12224 2026-04-12 18:49:59 -07:00 committed by GitHub
commit 5f26e48078
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
251 changed files with 10789 additions and 5443 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View file

@ -6,6 +6,15 @@ body:
attributes:
value: |
Thanks for taking the time to fill out this bug report!
- type: checkboxes
id: confirm-troubleshooting
attributes:
label: Have you enabled troubleshooting mode?
description: |
To ensure the bug is not caused by custom modifications or plugins make sure to enable troubleshooting mode before filing the report. In Stash go to Settings and click **Troubleshooting mode** and then retest to see if the bug still occurs.
options:
- label: I confirm that the troubleshooting mode is enabled.
required: true
- type: textarea
id: description
attributes:
@ -61,4 +70,4 @@ body:
attributes:
label: Relevant log output
description: Please copy and paste any relevant log output from Settings > Logs. This will be automatically formatted into code, so no need for backticks.
render: shell
render: shell

28
.github/workflows/build-compiler.yml vendored Normal file
View file

@ -0,0 +1,28 @@
name: Compiler Build
on:
workflow_dispatch:
env:
COMPILER_IMAGE: ghcr.io/stashapp/compiler:13
jobs:
build-compiler:
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/setup-buildx-action@v3
- uses: docker/build-push-action@v6
with:
push: true
context: "{{defaultContext}}:docker/compiler"
tags: |
${{ env.COMPILER_IMAGE }}
ghcr.io/stashapp/compiler:latest
cache-from: type=gha,scope=all,mode=max
cache-to: type=gha,scope=all,mode=max

270
.github/workflows/build.yml vendored
View file

@ -2,7 +2,7 @@ name: Build
on:
push:
branches:
branches:
- develop
- master
- 'releases/**'
@ -15,50 +15,163 @@ concurrency:
cancel-in-progress: true
env:
COMPILER_IMAGE: stashapp/compiler:12
COMPILER_IMAGE: ghcr.io/stashapp/compiler:13
jobs:
build:
runs-on: ubuntu-22.04
# Job 1: Generate code and build UI
# Runs natively (no Docker) — go generate/gqlgen and node don't need cross-compilers.
# Produces artifacts (generated Go files + UI build) consumed by test and build jobs.
generate:
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v6
with:
fetch-depth: 0
fetch-tags: true
- name: Setup Go
uses: actions/setup-go@v6
- name: Checkout
run: git fetch --prune --unshallow --tags
# pnpm version is read from the packageManager field in package.json
# very broken (4.3, 4.4)
- name: Install pnpm
uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061
with:
package_json_file: ui/v2.5/package.json
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: '20'
cache: 'pnpm'
cache-dependency-path: ui/v2.5/pnpm-lock.yaml
- name: Install UI dependencies
run: cd ui/v2.5 && pnpm install --frozen-lockfile
- name: Generate
run: make generate
- name: Cache UI build
uses: actions/cache@v5
id: cache-ui
with:
path: ui/v2.5/build
key: ${{ runner.os }}-ui-build-${{ hashFiles('ui/v2.5/pnpm-lock.yaml', 'ui/v2.5/public/**', 'ui/v2.5/src/**', 'graphql/**/*.graphql') }}
- name: Validate UI
# skip UI validation for pull requests if UI is unchanged
if: ${{ github.event_name != 'pull_request' || steps.cache-ui.outputs.cache-hit != 'true' }}
run: make validate-ui
- name: Build UI
# skip UI build for pull requests if UI is unchanged (UI was cached)
if: ${{ github.event_name != 'pull_request' || steps.cache-ui.outputs.cache-hit != 'true' }}
run: make ui
# Bundle generated Go files + UI build for downstream jobs (test + build)
- name: Upload generated artifacts
uses: actions/upload-artifact@v7
with:
name: generated
retention-days: 1
path: |
internal/api/generated_exec.go
internal/api/generated_models.go
ui/v2.5/build/
ui/login/locales/
# Job 2: Integration tests
# Runs natively (no Docker) — only needs Go + GCC (for CGO/SQLite), both on ubuntu-22.04.
# Runs in parallel with the build matrix jobs.
test:
needs: generate
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
- name: Setup Go
uses: actions/setup-go@v5
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Pull compiler image
run: docker pull $COMPILER_IMAGE
- name: Cache node modules
uses: actions/cache@v3
env:
cache-name: cache-node_modules
# Places generated Go files + UI build into the working tree so the build compiles
- name: Download generated artifacts
uses: actions/download-artifact@v8
with:
path: ui/v2.5/node_modules
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('ui/v2.5/pnpm-lock.yaml') }}
name: generated
- name: Cache UI build
uses: actions/cache@v3
id: cache-ui
env:
cache-name: cache-ui
- name: Test Backend
run: make it
# Job 3: Cross-compile for all platforms
# Each platform gets its own runner and Docker container (ghcr.io/stashapp/compiler:13).
# Each build-cc-* make target is self-contained (sets its own GOOS/GOARCH/CC),
# so running them in separate containers is functionally identical to one container.
# Runs in parallel with the test job.
build:
needs: generate
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
include:
- platform: windows
make-target: build-cc-windows
artifact-paths: |
dist/stash-win.exe
tag: win
- platform: macos
make-target: build-cc-macos
artifact-paths: |
dist/stash-macos
dist/Stash.app.zip
tag: osx
- platform: linux
make-target: build-cc-linux
artifact-paths: |
dist/stash-linux
tag: linux
- platform: linux-arm64v8
make-target: build-cc-linux-arm64v8
artifact-paths: |
dist/stash-linux-arm64v8
tag: arm
- platform: linux-arm32v7
make-target: build-cc-linux-arm32v7
artifact-paths: |
dist/stash-linux-arm32v7
tag: arm
- platform: linux-arm32v6
make-target: build-cc-linux-arm32v6
artifact-paths: |
dist/stash-linux-arm32v6
tag: arm
- platform: freebsd
make-target: build-cc-freebsd
artifact-paths: |
dist/stash-freebsd
tag: freebsd
steps:
- uses: actions/checkout@v6
with:
path: ui/v2.5/build
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('ui/v2.5/pnpm-lock.yaml', 'ui/v2.5/public/**', 'ui/v2.5/src/**', 'graphql/**/*.graphql') }}
fetch-depth: 0
fetch-tags: true
- name: Cache go build
uses: actions/cache@v3
env:
# increment the number suffix to bump the cache
cache-name: cache-go-cache-1
- name: Download generated artifacts
uses: actions/download-artifact@v8
with:
name: generated
- name: Cache Go build
uses: actions/cache@v5
with:
path: .go-cache
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('go.mod', '**/go.sum') }}
key: ${{ runner.os }}-go-cache-${{ matrix.platform }}-${{ hashFiles('go.mod', '**/go.sum') }}
# kept seperate to test timings
- name: pull compiler image
run: docker pull $COMPILER_IMAGE
- name: Start build container
env:
@ -67,45 +180,50 @@ jobs:
mkdir -p .go-cache
docker run -d --name build --mount type=bind,source="$(pwd)",target=/stash,consistency=delegated --mount type=bind,source="$(pwd)/.go-cache",target=/root/.cache/go-build,consistency=delegated --env OFFICIAL_BUILD=${{ env.official-build }} -w /stash $COMPILER_IMAGE tail -f /dev/null
- name: Pre-install
run: docker exec -t build /bin/bash -c "make CI=1 pre-ui"
- name: Generate
run: docker exec -t build /bin/bash -c "make generate"
- name: Validate UI
# skip UI validation for pull requests if UI is unchanged
if: ${{ github.event_name != 'pull_request' || steps.cache-ui.outputs.cache-hit != 'true' }}
run: docker exec -t build /bin/bash -c "make validate-ui"
# Static validation happens in the linter workflow in parallel to this workflow
# Run Dynamic validation here, to make sure we pass all the projects integration tests
- name: Test Backend
run: docker exec -t build /bin/bash -c "make it"
- name: Build UI
# skip UI build for pull requests if UI is unchanged (UI was cached)
# this means that the build version/time may be incorrect if the UI is
# not changed in a pull request
if: ${{ github.event_name != 'pull_request' || steps.cache-ui.outputs.cache-hit != 'true' }}
run: docker exec -t build /bin/bash -c "make ui"
- name: Compile for all supported platforms
run: |
docker exec -t build /bin/bash -c "make build-cc-windows"
docker exec -t build /bin/bash -c "make build-cc-macos"
docker exec -t build /bin/bash -c "make build-cc-linux"
docker exec -t build /bin/bash -c "make build-cc-linux-arm64v8"
docker exec -t build /bin/bash -c "make build-cc-linux-arm32v7"
docker exec -t build /bin/bash -c "make build-cc-linux-arm32v6"
docker exec -t build /bin/bash -c "make build-cc-freebsd"
- name: Zip UI
run: docker exec -t build /bin/bash -c "make zip-ui"
- name: Build (${{ matrix.platform }})
run: docker exec -t build /bin/bash -c "make ${{ matrix.make-target }}"
- name: Cleanup build container
run: docker rm -f -v build
- name: Upload build artifact
uses: actions/upload-artifact@v7
with:
name: build-${{ matrix.platform }}
retention-days: 1
path: ${{ matrix.artifact-paths }}
# Job 4: Release
# Waits for both test and build to pass, then collects all platform artifacts
# into dist/ for checksums, GitHub releases, and multi-arch Docker push.
release:
needs: [test, build]
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
fetch-tags: true
# Downloads all artifacts (generated + 7 platform builds) into artifacts/ subdirectories
- name: Download all build artifacts
uses: actions/download-artifact@v8
with:
path: artifacts
# Reassemble platform binaries from matrix job artifacts into a single dist/ directory
# make sure that artifacts have executable bit set
# upload-artifact@v4 strips the common path prefix (dist/), so files are at the artifact root
- name: Collect binaries
run: |
mkdir -p dist
cp artifacts/build-*/* dist/
chmod +x dist/*
- name: Zip UI
run: |
cd artifacts/generated/ui/v2.5/build && zip -r ../../../../../dist/stash-ui.zip .
- name: Generate checksums
run: |
git describe --tags --exclude latest_develop | tee CHECKSUMS_SHA1
@ -116,7 +234,7 @@ jobs:
- name: Upload Windows binary
# only upload binaries for pull requests
if: ${{ github.event_name == 'pull_request' && github.base_ref != 'refs/heads/develop' && github.base_ref != 'refs/heads/master'}}
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v7
with:
name: stash-win.exe
path: dist/stash-win.exe
@ -124,15 +242,23 @@ jobs:
- name: Upload macOS binary
# only upload binaries for pull requests
if: ${{ github.event_name == 'pull_request' && github.base_ref != 'refs/heads/develop' && github.base_ref != 'refs/heads/master'}}
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v7
with:
name: stash-macos
path: dist/stash-macos
- name: Upload macOS bundle
# only upload binaries for pull requests
if: ${{ github.event_name == 'pull_request' && github.base_ref != 'refs/heads/develop' && github.base_ref != 'refs/heads/master'}}
uses: actions/upload-artifact@v7
with:
name: Stash.app.zip
path: dist/Stash.app.zip
- name: Upload Linux binary
# only upload binaries for pull requests
if: ${{ github.event_name == 'pull_request' && github.base_ref != 'refs/heads/develop' && github.base_ref != 'refs/heads/master'}}
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v7
with:
name: stash-linux
path: dist/stash-linux
@ -140,14 +266,14 @@ jobs:
- name: Upload UI
# only upload for pull requests
if: ${{ github.event_name == 'pull_request' && github.base_ref != 'refs/heads/develop' && github.base_ref != 'refs/heads/master'}}
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v7
with:
name: stash-ui.zip
path: dist/stash-ui.zip
- name: Update latest_develop tag
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }}
run : git tag -f latest_develop; git push -f --tags
run: git tag -f latest_develop; git push -f --tags
- name: Development Release
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }}
@ -197,7 +323,7 @@ jobs:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: |
docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64
docker run --rm --privileged tonistiigi/binfmt
docker info
docker buildx create --name builder --use
docker buildx inspect --bootstrap
@ -213,7 +339,7 @@ jobs:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: |
docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64
docker run --rm --privileged tonistiigi/binfmt
docker info
docker buildx create --name builder --use
docker buildx inspect --bootstrap

61
.github/workflows/golangci-lint.yml vendored
View file

@ -9,65 +9,20 @@ on:
- 'releases/**'
pull_request:
env:
COMPILER_IMAGE: stashapp/compiler:12
jobs:
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Checkout
run: git fetch --prune --unshallow --tags
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: Pull compiler image
run: docker pull $COMPILER_IMAGE
- name: Start build container
run: |
mkdir -p .go-cache
docker run -d --name build --mount type=bind,source="$(pwd)",target=/stash,consistency=delegated --mount type=bind,source="$(pwd)/.go-cache",target=/root/.cache/go-build,consistency=delegated -w /stash $COMPILER_IMAGE tail -f /dev/null
# no tags or depth needed for lint
- uses: actions/checkout@v6
- uses: actions/setup-go@v6
# generate-backend runs natively (just go generate + touch-ui) — no Docker needed
- name: Generate Backend
run: docker exec -t build /bin/bash -c "make generate-backend"
run: make generate-backend
## WARN
## using v1, update in a later PR
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v6
with:
# Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
version: latest
# Optional: working directory, useful for monorepos
# working-directory: somedir
# Optional: golangci-lint command line arguments.
#
# Note: By default, the `.golangci.yml` file should be at the root of the repository.
# The location of the configuration file can be changed by using `--config=`
args: --timeout=5m
# Optional: show only new issues if it's a pull request. The default value is `false`.
# only-new-issues: true
# Optional: if set to true, then all caching functionality will be completely disabled,
# takes precedence over all other caching options.
# skip-cache: true
# Optional: if set to true, then the action won't cache or restore ~/go/pkg.
# skip-pkg-cache: true
# Optional: if set to true, then the action won't cache or restore ~/.cache/go-build.
# skip-build-cache: true
# Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'.
# install-mode: "goinstall"
- name: Cleanup build container
run: docker rm -f -v build
uses: golangci/golangci-lint-action@v6

16
Makefile
View file

@ -50,7 +50,7 @@ export CGO_ENABLED := 1
# define COMPILER_IMAGE for cross-compilation docker container
ifndef COMPILER_IMAGE
COMPILER_IMAGE := stashapp/compiler:latest
COMPILER_IMAGE := ghcr.io/stashapp/compiler:latest
endif
.PHONY: release
@ -129,7 +129,7 @@ phasher: build-flags
# builds dynamically-linked debug binaries
.PHONY: build
build: stash phasher
build: stash
# builds dynamically-linked PIE release binaries
.PHONY: build-release
@ -187,8 +187,6 @@ build-cc-macos:
# Combine into universal binaries
lipo -create -output dist/stash-macos dist/stash-macos-intel dist/stash-macos-arm
rm dist/stash-macos-intel dist/stash-macos-arm
lipo -create -output dist/phasher-macos dist/phasher-macos-intel dist/phasher-macos-arm
rm dist/phasher-macos-intel dist/phasher-macos-arm
# Place into bundle and zip up
rm -rf dist/Stash.app
@ -198,6 +196,16 @@ build-cc-macos:
cd dist && rm -f Stash.app.zip && zip -r Stash.app.zip Stash.app
rm -rf dist/Stash.app
.PHONY: build-cc-macos-phasher
build-cc-macos-phasher:
make build-cc-macos-arm
make build-cc-macos-intel
# Combine into universal binaries
lipo -create -output dist/phasher-macos dist/phasher-macos-intel dist/phasher-macos-arm
rm dist/phasher-macos-intel dist/phasher-macos-arm
# do not bundle phasher
.PHONY: build-cc-freebsd
build-cc-freebsd: export GOOS := freebsd
build-cc-freebsd: export GOARCH := amd64

58
README.md
View file

@ -13,10 +13,10 @@
![Screenshot of Stash web application interface](docs/readme_assets/demo_image.png)
* Stash gathers information about videos in your collection from the internet, and is extensible through the use of community-built plugins for a large number of content producers and sites.
* Stash supports a wide variety of both video and image formats.
* You can tag videos and find them later.
* Stash provides statistics about performers, tags, studios and more.
- Stash gathers information about videos in your collection from the internet, and is extensible through the use of community-built plugins for a large number of content producers and sites.
- Stash supports a wide variety of both video and image formats.
- You can tag videos and find them later.
- Stash provides statistics about performers, tags, studios and more.
You can [watch a SFW demo video](https://vimeo.com/545323354) to see it in action.
@ -24,17 +24,19 @@ For further information you can consult the [documentation](https://docs.stashap
# Installing Stash
> [!tip]
Step-by-step instructions are available at [docs.stashapp.cc/installation](https://docs.stashapp.cc/installation/).
#### Windows Users:
As of version 0.27.0, Stash no longer supports _Windows 7, 8, Server 2008 and Server 2012._
At least Windows 10 or Server 2016 is required.
#### Mac Users:
As of version 0.29.0, Stash requires _macOS 11 Big Sur_ or later.
Stash can still be run through docker on older versions of macOS.
> [!important]
>**Windows Users**
>
>As of version 0.27.0, Stash no longer supports _Windows 7, 8, Server 2008 and Server 2012._
>At least Windows 10 or Server 2016 is required.
>
>**macOS Users**
>
> As of version 0.29.0, Stash requires _macOS 11 Big Sur_ or later.
> Stash can still be run through docker on older versions of macOS.
<img src="docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS | <img src="docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="docs/readme_assets/docker_logo.svg" width="100%" height="75"> Docker
:---:|:---:|:---:|:---:
@ -85,23 +87,23 @@ The badge below shows the current translation status of Stash across all support
Need help or want to get involved? Start with the documentation, then reach out to the community if you need further assistance.
- Documentation
- Official docs: https://docs.stashapp.cc - official guides guides and troubleshooting.
- In-app manual: press <kbd>Shift</kbd> + <kbd>?</kbd> in the app or view the manual online: https://docs.stashapp.cc/in-app-manual.
- FAQ: https://discourse.stashapp.cc/c/support/faq/28 - common questions and answers.
- Community wiki: https://discourse.stashapp.cc/tags/c/community-wiki/22/stash - guides, how-to’s and tips.
### Documentation
- [Official documentation](https://docs.stashapp.cc) - official guides guides and troubleshooting.
- [In-app manual](https://docs.stashapp.cc/in-app-manual) press <kbd>Shift</kbd> + <kbd>?</kbd> in the app or view the manual online.
- [FAQ](https://discourse.stashapp.cc/c/support/faq/28) - common questions and answers.
- [Community wiki](https://discourse.stashapp.cc/tags/c/community-wiki/22/stash) - guides, how-to’s and tips.
- Community & discussion
- Community forum: https://discourse.stashapp.cc - community support, feature requests and discussions.
- Discord: https://discord.gg/2TsNFKt - real-time chat and community support.
- GitHub discussions: https://github.com/stashapp/stash/discussions - community support and feature discussions.
- Lemmy community: https://discuss.online/c/stashapp - Reddit-style community space.
### Community & discussion
- [Community forum](https://discourse.stashapp.cc) - community support, feature requests and discussions.
- [Discord](https://discord.gg/2TsNFKt) - real-time chat and community support.
- [GitHub discussions](https://github.com/stashapp/stash/discussions) - community support and feature discussions.
- [Lemmy community](https://discuss.online/c/stashapp) - board-style community space.
- Community scrapers & plugins
- Metadata sources: https://docs.stashapp.cc/metadata-sources/
- Plugins: https://docs.stashapp.cc/plugins/
- Themes: https://docs.stashapp.cc/themes/
- Other projects: https://docs.stashapp.cc/other-projects/
### Community scrapers & plugins
- [Metadata sources](https://docs.stashapp.cc/metadata-sources/)
- [Plugins](https://docs.stashapp.cc/plugins/)
- [Themes](https://docs.stashapp.cc/themes/)
- [Other projects](https://docs.stashapp.cc/other-projects/)
# For Developers

2
docker/ci/x86_64/Dockerfile
View file

@ -12,7 +12,7 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then BIN=stash-linux-arm32v6; \
FROM --platform=$TARGETPLATFORM alpine:latest AS app
COPY --from=binary /stash /usr/bin/
RUN apk add --no-cache ca-certificates python3 py3-requests py3-requests-toolbelt py3-lxml py3-pip ffmpeg tzdata vips vips-tools \
RUN apk add --no-cache ca-certificates python3 py3-requests py3-requests-toolbelt py3-lxml py3-pip ffmpeg tzdata vips vips-tools vips-heif \
&& pip install --break-system-packages mechanicalsoup cloudscraper stashapp-tools
ENV STASH_CONFIG_FILE=/root/.stash/config.yml

1
docker/compiler/.gitignore vendored
View file

@ -1 +0,0 @@
*.sdk.tar.*

138
docker/compiler/Dockerfile
View file

@ -1,82 +1,86 @@
FROM golang:1.24.3
### OSXCROSS
FROM debian:bookworm AS osxcross
# add osxcross
WORKDIR /tmp/osxcross
ARG OSXCROSS_REVISION=5e1b71fcceb23952f3229995edca1b6231525b5b
ADD --checksum=sha256:d3f771bbc20612fea577b18a71be3af2eb5ad2dd44624196cf55de866d008647 https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REVISION} /tmp/osxcross.tar.gz
LABEL maintainer="https://discord.gg/2TsNFKt"
ARG OSX_SDK_VERSION=11.3
ARG OSX_SDK_DOWNLOAD_FILE=MacOSX${OSX_SDK_VERSION}.sdk.tar.xz
ARG OSX_SDK_DOWNLOAD_URL=https://github.com/phracker/MacOSX-SDKs/releases/download/${OSX_SDK_VERSION}/${OSX_SDK_DOWNLOAD_FILE}
ADD --checksum=sha256:cd4f08a75577145b8f05245a2975f7c81401d75e9535dcffbb879ee1deefcbf4 ${OSX_SDK_DOWNLOAD_URL} /tmp/osxcross/tarballs/${OSX_SDK_DOWNLOAD_FILE}
RUN apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg
ENV UNATTENDED=yes \
SDK_VERSION=${OSX_SDK_VERSION} \
OSX_VERSION_MIN=10.10
RUN apt update && \
apt install -y --no-install-recommends \
bash ca-certificates clang cmake git patch libssl-dev bzip2 cpio libbz2-dev libxml2-dev make python3 xz-utils zlib1g-dev
# lzma-dev libxml2-dev xz
RUN tar --strip=1 -C /tmp/osxcross -xf /tmp/osxcross.tar.gz
RUN ./build.sh
RUN mkdir -p /etc/apt/keyrings
### FREEBSD cross-compilation stage
# use alpine for cacheable image since apt is notorous for not caching
FROM alpine:3 AS freebsd
# match golang latest
# https://go.dev/wiki/FreeBSD
ARG FREEBSD_VERSION=12.4
ADD --checksum=sha256:581c7edacfd2fca2bdf5791f667402d22fccd8a5e184635e0cac075564d57aa8 \
http://ftp-archive.freebsd.org/mirror/FreeBSD-Archive/old-releases/amd64/${FREEBSD_VERSION}-RELEASE/base.txz \
/tmp/base.txz
ADD https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key nodesource.gpg.key
RUN cat nodesource.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && rm nodesource.gpg.key
RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
WORKDIR /opt/cross-freebsd
RUN apk add --no-cache tar xz
RUN tar -xf /tmp/base.txz --strip-components=1 ./usr/lib ./usr/include ./lib
RUN cd /opt/cross-freebsd/usr/lib && \
find . -type l -exec sh -c ' \
for link; do \
target=$(readlink "$link"); \
case "$target" in \
/lib/*) ln -sf "/opt/cross-freebsd$target" "$link";; \
esac; \
done \
' sh {} + && \
ln -s libc++.a libstdc++.a && \
ln -s libc++.so libstdc++.so
RUN apt-get update && \
apt-get install -y --no-install-recommends \
git make tar bash nodejs zip \
clang llvm-dev cmake patch libxml2-dev uuid-dev libssl-dev xz-utils \
bzip2 gzip sed cpio libbz2-dev zlib1g-dev \
gcc-mingw-w64 \
gcc-arm-linux-gnueabi libc-dev-armel-cross linux-libc-dev-armel-cross \
gcc-aarch64-linux-gnu libc-dev-arm64-cross && \
rm -rf /var/lib/apt/lists/*;
### BUILDER
FROM golang:1.24.3 AS builder
ENV PATH=/opt/osx-ndk-x86/bin:$PATH
# copy in nodejs instead of using nodesource :thumbsup:
COPY --from=docker.io/library/node:24-bookworm /usr/local /usr/local
# copy in osxcross
COPY --from=osxcross /tmp/osxcross/target/lib /usr/lib
COPY --from=osxcross /tmp/osxcross/target /opt/osx-ndk-x86
# copy in cross-freebsd
COPY --from=freebsd /opt/cross-freebsd /opt/cross-freebsd
# pnpm install with npm
RUN npm install -g pnpm
# FreeBSD cross-compilation setup
# https://github.com/smartmontools/docker-build/blob/6b8c92560d17d325310ba02d9f5a4b250cb0764a/Dockerfile#L66
ENV FREEBSD_VERSION 13.4
ENV FREEBSD_DOWNLOAD_URL http://ftp.plusline.de/FreeBSD/releases/amd64/${FREEBSD_VERSION}-RELEASE/base.txz
ENV FREEBSD_SHA 8e13b0a93daba349b8d28ad246d7beb327659b2ef4fe44d89f447392daec5a7c
# git for getting hash
# make and bash for building
RUN cd /tmp && \
curl -o base.txz $FREEBSD_DOWNLOAD_URL && \
echo "$FREEBSD_SHA base.txz" | sha256sum -c - && \
mkdir -p /opt/cross-freebsd && \
cd /opt/cross-freebsd && \
tar -xf /tmp/base.txz ./lib/ ./usr/lib/ ./usr/include/ && \
rm -f /tmp/base.txz && \
cd /opt/cross-freebsd/usr/lib && \
find . -xtype l | xargs ls -l | grep ' /lib/' | awk '{print "ln -sf /opt/cross-freebsd"$11 " " $9}' | /bin/sh && \
ln -s libc++.a libstdc++.a && \
ln -s libc++.so libstdc++.so
# macOS cross-compilation setup
ENV OSX_SDK_VERSION 11.3
ENV OSX_SDK_DOWNLOAD_FILE MacOSX${OSX_SDK_VERSION}.sdk.tar.xz
ENV OSX_SDK_DOWNLOAD_URL https://github.com/phracker/MacOSX-SDKs/releases/download/${OSX_SDK_VERSION}/${OSX_SDK_DOWNLOAD_FILE}
ENV OSX_SDK_SHA cd4f08a75577145b8f05245a2975f7c81401d75e9535dcffbb879ee1deefcbf4
ENV OSXCROSS_REVISION 5e1b71fcceb23952f3229995edca1b6231525b5b
ENV OSXCROSS_DOWNLOAD_URL https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REVISION}
ENV OSXCROSS_SHA d3f771bbc20612fea577b18a71be3af2eb5ad2dd44624196cf55de866d008647
RUN cd /tmp && \
curl -o osxcross.tar.gz $OSXCROSS_DOWNLOAD_URL && \
echo "$OSXCROSS_SHA osxcross.tar.gz" | sha256sum -c - && \
mkdir osxcross && \
tar --strip=1 -C osxcross -xf osxcross.tar.gz && \
rm -f osxcross.tar.gz && \
curl -Lo $OSX_SDK_DOWNLOAD_FILE $OSX_SDK_DOWNLOAD_URL && \
echo "$OSX_SDK_SHA $OSX_SDK_DOWNLOAD_FILE" | sha256sum -c - && \
mv $OSX_SDK_DOWNLOAD_FILE osxcross/tarballs/ && \
UNATTENDED=yes SDK_VERSION=$OSX_SDK_VERSION OSX_VERSION_MIN=10.10 osxcross/build.sh && \
cp osxcross/target/lib/* /usr/lib/ && \
mv osxcross/target /opt/osx-ndk-x86 && \
rm -rf /tmp/osxcross
ENV PATH /opt/osx-ndk-x86/bin:$PATH
RUN mkdir -p /root/.ssh && \
chmod 0700 /root/.ssh && \
ssh-keyscan github.com > /root/.ssh/known_hosts
# ignore "dubious ownership" errors
# clang for macos
# zip for stashapp.zip
# gcc-extensions for cross-arch build
# we still target arm soft float?
RUN apt-get update && \
apt-get install -y --no-install-recommends \
git make bash \
clang zip \
gcc-mingw-w64 \
gcc-arm-linux-gnueabi \
libc-dev-armel-cross linux-libc-dev-armel-cross \
gcc-aarch64-linux-gnu libc-dev-arm64-cross && \
rm -rf /var/lib/apt/lists/*;
RUN git config --global safe.directory '*'
# To test locally:
# make generate
# make ui
# cd docker/compiler
# make build
# docker run --rm -v /PATH_TO_STASH:/stash -w /stash -i -t stashapp/compiler:latest make build-cc-all
# # binaries will show up in /dist
# docker build . -t ghcr.io/stashapp/compiler:latest
# docker run --rm -v /PATH_TO_STASH:/stash -w /stash -i -t ghcr.io/stashapp/compiler:latest make build-cc-all
# # binaries will show up in /dist

20
docker/compiler/Makefile
View file

@ -1,16 +1,22 @@
host=ghcr.io
user=stashapp
repo=compiler
version=12
version=13
VERSION_IMAGE = ${host}/${user}/${repo}:${version}
LATEST_IMAGE = ${host}/${user}/${repo}:latest
latest:
docker build -t ${user}/${repo}:latest .
docker build -t ${LATEST_IMAGE} .
build:
docker build -t ${user}/${repo}:${version} -t ${user}/${repo}:latest .
docker build -t ${VERSION_IMAGE} -t ${LATEST_IMAGE} .
build-no-cache:
docker build --no-cache -t ${user}/${repo}:${version} -t ${user}/${repo}:latest .
docker build --no-cache -t ${VERSION_IMAGE} -t ${LATEST_IMAGE} .
install: build
docker push ${user}/${repo}:${version}
docker push ${user}/${repo}:latest
# requires docker login ghcr.io
# echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin
push:
docker push ${VERSION_IMAGE}
docker push ${LATEST_IMAGE}

2
docker/compiler/README.md
View file

@ -1,3 +1,3 @@
Modified from https://github.com/bep/dockerfiles/tree/master/ci-goreleaser
When the Dockerfile is changed, the version number should be incremented in the Makefile and the new version tag should be pushed to Docker Hub. The GitHub workflow files also need to be updated to pull the correct image tag.
When the Dockerfile is changed, the version number should be incremented in [.github/workflows/build-compiler.yml](../../.github/workflows/build-compiler.yml) and the workflow [manually ran](). `env: COMPILER_IMAGE` in [.github/workflows/build.yml](../../.github/workflows/build.yml) also needs to be updated to pull the correct image tag.

4
docs/DEVELOPMENT.md
View file

@ -118,8 +118,8 @@ This project uses a modification of the [CI-GoReleaser](https://github.com/bep/d
To cross-compile the app yourself:
1. Run `make pre-ui`, `make generate` and `make ui` outside the container, to generate files and build the UI.
2. Pull the latest compiler image from Docker Hub: `docker pull stashapp/compiler`
3. Run `docker run --rm --mount type=bind,source="$(pwd)",target=/stash -w /stash -it stashapp/compiler /bin/bash` to open a shell inside the container.
2. Pull the latest compiler image from GHCR: `docker pull ghcr.io/stashapp/compiler`
3. Run `docker run --rm --mount type=bind,source="$(pwd)",target=/stash -w /stash -it ghcr.io/stashapp/compiler /bin/bash` to open a shell inside the container.
4. From inside the container, run `make build-cc-all` to build for all platforms, or run `make build-cc-{platform}` to build for a specific platform (have a look at the `Makefile` for the list of targets).
5. You will find the compiled binaries in `dist/`.

5
graphql/schema/types/file.graphql
View file

@ -16,6 +16,9 @@ type Folder {
parent_folders: [Folder!]!
zip_file: BasicFile
"Returns direct sub-folders"
sub_folders: [Folder!]!
mod_time: Time!
created_at: Time!
@ -156,7 +159,7 @@ input MoveFilesInput {
input SetFingerprintsInput {
type: String!
"an null value will remove the fingerprint"
"a null value will remove the fingerprint"
value: String
}

18
graphql/schema/types/filters.graphql
View file

@ -152,15 +152,15 @@ input PerformerFilterType {
fake_tits: StringCriterionInput
"Filter by penis length value"
penis_length: FloatCriterionInput
"Filter by ciricumcision"
"Filter by circumcision"
circumcised: CircumcisionCriterionInput
"Deprecated: use career_start and career_end. This filter is non-functional."
career_length: StringCriterionInput
@deprecated(reason: "Use career_start and career_end")
"Filter by career start year"
career_start: IntCriterionInput
"Filter by career end year"
career_end: IntCriterionInput
"Filter by career start"
career_start: DateCriterionInput
"Filter by career end"
career_end: DateCriterionInput
"Filter by tattoos"
tattoos: StringCriterionInput
"Filter by piercings"
@ -249,9 +249,9 @@ input SceneMarkerFilterType {
updated_at: TimestampCriterionInput
"Filter by scene date"
scene_date: DateCriterionInput
"Filter by cscene reation time"
"Filter by scene creation time"
scene_created_at: TimestampCriterionInput
"Filter by lscene ast update time"
"Filter by scene last update time"
scene_updated_at: TimestampCriterionInput
"Filter by related scenes that meet this criteria"
scene_filter: SceneFilterType
@ -665,7 +665,7 @@ input TagFilterType {
"Filter by number of parent tags the tag has"
parent_count: IntCriterionInput
"Filter by number f child tags the tag has"
"Filter by number of child tags the tag has"
child_count: IntCriterionInput
"Filter by autotag ignore value"
@ -933,7 +933,7 @@ input GenderCriterionInput {
}
input CircumcisionCriterionInput {
value: [CircumisedEnum!]
value: [CircumcisedEnum!]
modifier: CriterionModifier!
}

2
graphql/schema/types/group.graphql
View file

@ -99,6 +99,8 @@ input BulkGroupUpdateInput {
ids: [ID!]
# rating expressed as 1-100
rating100: Int
date: String
synopsis: String
studio_id: ID
director: String
urls: BulkUpdateStrings

8
graphql/schema/types/metadata.graphql
View file

@ -131,6 +131,14 @@ type ScanMetadataOptions {
input CleanMetadataInput {
paths: [String!]
"""
Don't check zip file contents when determining whether to clean a file.
This can significantly speed up the clean process, but will potentially miss removed files within zip files.
Where users do not modify zip files contents directly, this should be safe to use.
Defaults to false.
"""
ignoreZipFileContents: Boolean
"Do a dry run. Don't delete any files"
dryRun: Boolean!
}

26
graphql/schema/types/performer.graphql
View file

@ -7,7 +7,7 @@ enum GenderEnum {
NON_BINARY
}
enum CircumisedEnum {
enum CircumcisedEnum {
CUT
UNCUT
}
@ -29,10 +29,10 @@ type Performer {
measurements: String
fake_tits: String
penis_length: Float
circumcised: CircumisedEnum
circumcised: CircumcisedEnum
career_length: String @deprecated(reason: "Use career_start and career_end")
career_start: Int
career_end: Int
career_start: String
career_end: String
tattoos: String
piercings: String
alias_list: [String!]!
@ -78,10 +78,10 @@ input PerformerCreateInput {
measurements: String
fake_tits: String
penis_length: Float
circumcised: CircumisedEnum
circumcised: CircumcisedEnum
career_length: String @deprecated(reason: "Use career_start and career_end")
career_start: Int
career_end: Int
career_start: String
career_end: String
tattoos: String
piercings: String
"Duplicate aliases and those equal to name will be ignored (case-insensitive)"
@ -119,10 +119,10 @@ input PerformerUpdateInput {
measurements: String
fake_tits: String
penis_length: Float
circumcised: CircumisedEnum
circumcised: CircumcisedEnum
career_length: String @deprecated(reason: "Use career_start and career_end")
career_start: Int
career_end: Int
career_start: String
career_end: String
tattoos: String
piercings: String
"Duplicate aliases and those equal to name will be ignored (case-insensitive)"
@ -165,10 +165,10 @@ input BulkPerformerUpdateInput {
measurements: String
fake_tits: String
penis_length: Float
circumcised: CircumisedEnum
circumcised: CircumcisedEnum
career_length: String @deprecated(reason: "Use career_start and career_end")
career_start: Int
career_end: Int
career_start: String
career_end: String
tattoos: String
piercings: String
"Duplicate aliases and those equal to name will result in an error (case-insensitive)"

8
graphql/schema/types/scraped-performer.graphql
View file

@ -19,8 +19,8 @@ type ScrapedPerformer {
penis_length: String
circumcised: String
career_length: String @deprecated(reason: "Use career_start and career_end")
career_start: Int
career_end: Int
career_start: String
career_end: String
tattoos: String
piercings: String
# aliases must be comma-delimited to be parsed correctly
@ -57,8 +57,8 @@ input ScrapedPerformerInput {
penis_length: String
circumcised: String
career_length: String @deprecated(reason: "Use career_start and career_end")
career_start: Int
career_end: Int
career_start: String
career_end: String
tattoos: String
piercings: String
aliases: String

1
graphql/schema/types/scraper.graphql
View file

@ -73,6 +73,7 @@ type ScrapedTag {
name: String!
description: String
alias_list: [String!]
parent: ScrapedTag
"Remote site ID, if applicable"
remote_site_id: String
}

5
graphql/stash-box/query.graphql
View file

@ -31,6 +31,11 @@ fragment TagFragment on Tag {
id
description
aliases
category {
id
name
description
}
}
fragment MeasurementsFragment on Measurements {

23
internal/api/loaders/dataloaders.go
View file

@ -11,7 +11,7 @@
//go:generate go run github.com/vektah/dataloaden GroupLoader int *github.com/stashapp/stash/pkg/models.Group
//go:generate go run github.com/vektah/dataloaden FileLoader github.com/stashapp/stash/pkg/models.FileID github.com/stashapp/stash/pkg/models.File
//go:generate go run github.com/vektah/dataloaden FolderLoader github.com/stashapp/stash/pkg/models.FolderID *github.com/stashapp/stash/pkg/models.Folder
//go:generate go run github.com/vektah/dataloaden FolderParentFolderIDsLoader github.com/stashapp/stash/pkg/models.FolderID []github.com/stashapp/stash/pkg/models.FolderID
//go:generate go run github.com/vektah/dataloaden FolderRelatedFolderIDsLoader github.com/stashapp/stash/pkg/models.FolderID []github.com/stashapp/stash/pkg/models.FolderID
//go:generate go run github.com/vektah/dataloaden SceneFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID
//go:generate go run github.com/vektah/dataloaden ImageFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID
//go:generate go run github.com/vektah/dataloaden GalleryFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID
@ -75,7 +75,8 @@ type Loaders struct {
FileByID *FileLoader
FolderByID *FolderLoader
FolderParentFolderIDs *FolderParentFolderIDsLoader
FolderParentFolderIDs *FolderRelatedFolderIDsLoader
FolderSubFolderIDs *FolderRelatedFolderIDsLoader
}
type Middleware struct {
@ -166,11 +167,16 @@ func (m Middleware) Middleware(next http.Handler) http.Handler {
maxBatch: maxBatch,
fetch: m.fetchFolders(ctx),
},
FolderParentFolderIDs: &FolderParentFolderIDsLoader{
FolderParentFolderIDs: &FolderRelatedFolderIDsLoader{
wait: wait,
maxBatch: maxBatch,
fetch: m.fetchFoldersParentFolderIDs(ctx),
},
FolderSubFolderIDs: &FolderRelatedFolderIDsLoader{
wait: wait,
maxBatch: maxBatch,
fetch: m.fetchFoldersSubFolderIDs(ctx),
},
SceneFiles: &SceneFileIDsLoader{
wait: wait,
maxBatch: maxBatch,
@ -427,6 +433,17 @@ func (m Middleware) fetchFoldersParentFolderIDs(ctx context.Context) func(keys [
}
}
func (m Middleware) fetchFoldersSubFolderIDs(ctx context.Context) func(keys []models.FolderID) ([][]models.FolderID, []error) {
return func(keys []models.FolderID) (ret [][]models.FolderID, errs []error) {
err := m.Repository.WithDB(ctx, func(ctx context.Context) error {
var err error
ret, err = m.Repository.Folder.GetManySubFolderIDs(ctx, keys)
return err
})
return ret, toErrorSlice(err)
}
}
func (m Middleware) fetchScenesFileIDs(ctx context.Context) func(keys []int) ([][]models.FileID, []error) {
return func(keys []int) (ret [][]models.FileID, errs []error) {
err := m.Repository.WithDB(ctx, func(ctx context.Context) error {

28
internal/api/loaders/folderparentfolderidsloader_gen.go → internal/api/loaders/folderrelatedfolderidsloader_gen.go
View file

@ -22,16 +22,16 @@ type FolderParentFolderIDsLoaderConfig struct {
}
// NewFolderParentFolderIDsLoader creates a new FolderParentFolderIDsLoader given a fetch, wait, and maxBatch
func NewFolderParentFolderIDsLoader(config FolderParentFolderIDsLoaderConfig) *FolderParentFolderIDsLoader {
return &FolderParentFolderIDsLoader{
func NewFolderParentFolderIDsLoader(config FolderParentFolderIDsLoaderConfig) *FolderRelatedFolderIDsLoader {
return &FolderRelatedFolderIDsLoader{
fetch: config.Fetch,
wait: config.Wait,
maxBatch: config.MaxBatch,
}
}
// FolderParentFolderIDsLoader batches and caches requests
type FolderParentFolderIDsLoader struct {
// FolderRelatedFolderIDsLoader batches and caches requests
type FolderRelatedFolderIDsLoader struct {
// this method provides the data for the loader
fetch func(keys []models.FolderID) ([][]models.FolderID, []error)
@ -63,14 +63,14 @@ type folderParentFolderIDsLoaderBatch struct {
}
// Load a FolderID by key, batching and caching will be applied automatically
func (l *FolderParentFolderIDsLoader) Load(key models.FolderID) ([]models.FolderID, error) {
func (l *FolderRelatedFolderIDsLoader) Load(key models.FolderID) ([]models.FolderID, error) {
return l.LoadThunk(key)()
}
// LoadThunk returns a function that when called will block waiting for a FolderID.
// This method should be used if you want one goroutine to make requests to many
// different data loaders without blocking until the thunk is called.
func (l *FolderParentFolderIDsLoader) LoadThunk(key models.FolderID) func() ([]models.FolderID, error) {
func (l *FolderRelatedFolderIDsLoader) LoadThunk(key models.FolderID) func() ([]models.FolderID, error) {
l.mu.Lock()
if it, ok := l.cache[key]; ok {
l.mu.Unlock()
@ -113,7 +113,7 @@ func (l *FolderParentFolderIDsLoader) LoadThunk(key models.FolderID) func() ([]m
// LoadAll fetches many keys at once. It will be broken into appropriate sized
// sub batches depending on how the loader is configured
func (l *FolderParentFolderIDsLoader) LoadAll(keys []models.FolderID) ([][]models.FolderID, []error) {
func (l *FolderRelatedFolderIDsLoader) LoadAll(keys []models.FolderID) ([][]models.FolderID, []error) {
results := make([]func() ([]models.FolderID, error), len(keys))
for i, key := range keys {
@ -131,7 +131,7 @@ func (l *FolderParentFolderIDsLoader) LoadAll(keys []models.FolderID) ([][]model
// LoadAllThunk returns a function that when called will block waiting for a FolderIDs.
// This method should be used if you want one goroutine to make requests to many
// different data loaders without blocking until the thunk is called.
func (l *FolderParentFolderIDsLoader) LoadAllThunk(keys []models.FolderID) func() ([][]models.FolderID, []error) {
func (l *FolderRelatedFolderIDsLoader) LoadAllThunk(keys []models.FolderID) func() ([][]models.FolderID, []error) {
results := make([]func() ([]models.FolderID, error), len(keys))
for i, key := range keys {
results[i] = l.LoadThunk(key)
@ -149,7 +149,7 @@ func (l *FolderParentFolderIDsLoader) LoadAllThunk(keys []models.FolderID) func(
// Prime the cache with the provided key and value. If the key already exists, no change is made
// and false is returned.
// (To forcefully prime the cache, clear the key first with loader.clear(key).prime(key, value).)
func (l *FolderParentFolderIDsLoader) Prime(key models.FolderID, value []models.FolderID) bool {
func (l *FolderRelatedFolderIDsLoader) Prime(key models.FolderID, value []models.FolderID) bool {
l.mu.Lock()
var found bool
if _, found = l.cache[key]; !found {
@ -164,13 +164,13 @@ func (l *FolderParentFolderIDsLoader) Prime(key models.FolderID, value []models.
}
// Clear the value at key from the cache, if it exists
func (l *FolderParentFolderIDsLoader) Clear(key models.FolderID) {
func (l *FolderRelatedFolderIDsLoader) Clear(key models.FolderID) {
l.mu.Lock()
delete(l.cache, key)
l.mu.Unlock()
}
func (l *FolderParentFolderIDsLoader) unsafeSet(key models.FolderID, value []models.FolderID) {
func (l *FolderRelatedFolderIDsLoader) unsafeSet(key models.FolderID, value []models.FolderID) {
if l.cache == nil {
l.cache = map[models.FolderID][]models.FolderID{}
}
@ -179,7 +179,7 @@ func (l *FolderParentFolderIDsLoader) unsafeSet(key models.FolderID, value []mod
// keyIndex will return the location of the key in the batch, if its not found
// it will add the key to the batch
func (b *folderParentFolderIDsLoaderBatch) keyIndex(l *FolderParentFolderIDsLoader, key models.FolderID) int {
func (b *folderParentFolderIDsLoaderBatch) keyIndex(l *FolderRelatedFolderIDsLoader, key models.FolderID) int {
for i, existingKey := range b.keys {
if key == existingKey {
return i
@ -203,7 +203,7 @@ func (b *folderParentFolderIDsLoaderBatch) keyIndex(l *FolderParentFolderIDsLoad
return pos
}
func (b *folderParentFolderIDsLoaderBatch) startTimer(l *FolderParentFolderIDsLoader) {
func (b *folderParentFolderIDsLoaderBatch) startTimer(l *FolderRelatedFolderIDsLoader) {
time.Sleep(l.wait)
l.mu.Lock()
@ -219,7 +219,7 @@ func (b *folderParentFolderIDsLoaderBatch) startTimer(l *FolderParentFolderIDsLo
b.end(l)
}
func (b *folderParentFolderIDsLoaderBatch) end(l *FolderParentFolderIDsLoader) {
func (b *folderParentFolderIDsLoaderBatch) end(l *FolderRelatedFolderIDsLoader) {
b.data, b.error = l.fetch(b.keys)
close(b.done)
}

8
internal/api/resolver.go
View file

@ -7,6 +7,7 @@ import (
"sort"
"strconv"
"github.com/99designs/gqlgen/graphql"
"github.com/stashapp/stash/internal/build"
"github.com/stashapp/stash/internal/manager"
"github.com/stashapp/stash/pkg/logger"
@ -145,6 +146,13 @@ func (r *Resolver) withReadTxn(ctx context.Context, fn func(ctx context.Context)
return r.repository.WithReadTxn(ctx, fn)
}
// idOnly returns true if the query is only asking for the id field.
// This can be used to optimize certain queries where we don't need to load the full object if we're only getting the id.
func (r *Resolver) idOnly(ctx context.Context) bool {
fields := graphql.CollectAllFields(ctx)
return len(fields) == 1 && fields[0] == "id"
}
func (r *queryResolver) MarkerWall(ctx context.Context, q *string) (ret []*models.SceneMarker, err error) {
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
ret, err = r.repository.SceneMarker.Wall(ctx, q)

42
internal/api/resolver_model_folder.go
View file

@ -17,20 +17,62 @@ func (r *folderResolver) ParentFolder(ctx context.Context, obj *models.Folder) (
return nil, nil
}
if r.idOnly(ctx) {
return &models.Folder{ID: *obj.ParentFolderID}, nil
}
return loaders.From(ctx).FolderByID.Load(*obj.ParentFolderID)
}
func foldersFromIDs(ids []models.FolderID) []*models.Folder {
ret := make([]*models.Folder, len(ids))
for i, id := range ids {
ret[i] = &models.Folder{ID: id}
}
return ret
}
func (r *folderResolver) ParentFolders(ctx context.Context, obj *models.Folder) ([]*models.Folder, error) {
ids, err := loaders.From(ctx).FolderParentFolderIDs.Load(obj.ID)
if err != nil {
return nil, err
}
if r.idOnly(ctx) {
return foldersFromIDs(ids), nil
}
var errs []error
ret, errs := loaders.From(ctx).FolderByID.LoadAll(ids)
return ret, firstError(errs)
}
func (r *folderResolver) SubFolders(ctx context.Context, obj *models.Folder) ([]*models.Folder, error) {
ids, err := loaders.From(ctx).FolderSubFolderIDs.Load(obj.ID)
if err != nil {
return nil, err
}
if r.idOnly(ctx) {
return foldersFromIDs(ids), nil
}
var errs []error
ret, errs := loaders.From(ctx).FolderByID.LoadAll(ids)
return ret, firstError(errs)
}
func (r *folderResolver) ZipFile(ctx context.Context, obj *models.Folder) (*BasicFile, error) {
// shortcut for id only queries
if r.idOnly(ctx) {
if obj.ZipFileID == nil {
return nil, nil
}
return &BasicFile{
BaseFile: &models.BaseFile{ID: *obj.ZipFileID},
}, nil
}
return zipFileResolver(ctx, obj.ZipFileID)
}

19
internal/api/resolver_model_performer.go
View file

@ -10,7 +10,6 @@ import (
"github.com/stashapp/stash/pkg/image"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/performer"
"github.com/stashapp/stash/pkg/utils"
)
func (r *performerResolver) AliasList(ctx context.Context, obj *models.Performer) ([]string, error) {
@ -110,12 +109,28 @@ func (r *performerResolver) HeightCm(ctx context.Context, obj *models.Performer)
return obj.Height, nil
}
func (r *performerResolver) CareerStart(ctx context.Context, obj *models.Performer) (*string, error) {
if obj.CareerStart != nil {
ret := obj.CareerStart.String()
return &ret, nil
}
return nil, nil
}
func (r *performerResolver) CareerEnd(ctx context.Context, obj *models.Performer) (*string, error) {
if obj.CareerEnd != nil {
ret := obj.CareerEnd.String()
return &ret, nil
}
return nil, nil
}
func (r *performerResolver) CareerLength(ctx context.Context, obj *models.Performer) (*string, error) {
if obj.CareerStart == nil && obj.CareerEnd == nil {
return nil, nil
}
ret := utils.FormatYearRange(obj.CareerStart, obj.CareerEnd)
ret := models.FormatYearRange(obj.CareerStart, obj.CareerEnd)
return &ret, nil
}

6
internal/api/resolver_mutation_group.go
View file

@ -227,6 +227,12 @@ func (r *mutationResolver) GroupUpdate(ctx context.Context, input GroupUpdateInp
func groupPartialFromBulkGroupUpdateInput(translator changesetTranslator, input BulkGroupUpdateInput) (ret models.GroupPartial, err error) {
updatedGroup := models.NewGroupPartial()
updatedGroup.Date, err = translator.optionalDate(input.Date, "date")
if err != nil {
err = fmt.Errorf("converting date: %w", err)
return
}
updatedGroup.Synopsis = translator.optionalString(input.Synopsis, "synopsis")
updatedGroup.Rating = translator.optionalInt(input.Rating100, "rating100")
updatedGroup.Director = translator.optionalString(input.Director, "director")

4
internal/api/resolver_mutation_migrate.go
View file

@ -47,6 +47,10 @@ func (r *mutationResolver) Migrate(ctx context.Context, input manager.MigrateInp
Database: mgr.Database,
}
if err := t.PreExecute(); err != nil {
return "", err
}
jobID := mgr.JobManager.Add(ctx, "Migrating database...", t)
return strconv.Itoa(jobID), nil

63
internal/api/resolver_mutation_performer.go
View file

@ -52,17 +52,6 @@ func (r *mutationResolver) PerformerCreate(ctx context.Context, input models.Per
newPerformer.FakeTits = translator.string(input.FakeTits)
newPerformer.PenisLength = input.PenisLength
newPerformer.Circumcised = input.Circumcised
newPerformer.CareerStart = input.CareerStart
newPerformer.CareerEnd = input.CareerEnd
// if career_start/career_end not provided, parse deprecated career_length
if newPerformer.CareerStart == nil && newPerformer.CareerEnd == nil && input.CareerLength != nil {
start, end, err := utils.ParseYearRangeString(*input.CareerLength)
if err != nil {
return nil, fmt.Errorf("could not parse career_length %q: %w", *input.CareerLength, err)
}
newPerformer.CareerStart = start
newPerformer.CareerEnd = end
}
newPerformer.Tattoos = translator.string(input.Tattoos)
newPerformer.Piercings = translator.string(input.Piercings)
newPerformer.Favorite = translator.bool(input.Favorite)
@ -100,6 +89,25 @@ func (r *mutationResolver) PerformerCreate(ctx context.Context, input models.Per
return nil, fmt.Errorf("converting death date: %w", err)
}
newPerformer.CareerStart, err = translator.datePtr(input.CareerStart)
if err != nil {
return nil, fmt.Errorf("converting career start: %w", err)
}
newPerformer.CareerEnd, err = translator.datePtr(input.CareerEnd)
if err != nil {
return nil, fmt.Errorf("converting career end: %w", err)
}
// if career_start/career_end not provided, parse deprecated career_length
if newPerformer.CareerStart == nil && newPerformer.CareerEnd == nil && input.CareerLength != nil {
start, end, err := models.ParseYearRangeString(*input.CareerLength)
if err != nil {
return nil, fmt.Errorf("could not parse career_length %q: %w", *input.CareerLength, err)
}
newPerformer.CareerStart = start
newPerformer.CareerEnd = end
}
newPerformer.TagIDs, err = translator.relatedIds(input.TagIds)
if err != nil {
return nil, fmt.Errorf("converting tag ids: %w", err)
@ -273,18 +281,25 @@ func performerPartialFromInput(input models.PerformerUpdateInput, translator cha
updatedPerformer.Circumcised = translator.optionalString((*string)(input.Circumcised), "circumcised")
// prefer career_start/career_end over deprecated career_length
if translator.hasField("career_start") || translator.hasField("career_end") {
updatedPerformer.CareerStart = translator.optionalInt(input.CareerStart, "career_start")
updatedPerformer.CareerEnd = translator.optionalInt(input.CareerEnd, "career_end")
var err error
updatedPerformer.CareerStart, err = translator.optionalDate(input.CareerStart, "career_start")
if err != nil {
return nil, fmt.Errorf("converting career start: %w", err)
}
updatedPerformer.CareerEnd, err = translator.optionalDate(input.CareerEnd, "career_end")
if err != nil {
return nil, fmt.Errorf("converting career end: %w", err)
}
} else if translator.hasField("career_length") && input.CareerLength != nil {
start, end, err := utils.ParseYearRangeString(*input.CareerLength)
start, end, err := models.ParseYearRangeString(*input.CareerLength)
if err != nil {
return nil, fmt.Errorf("could not parse career_length %q: %w", *input.CareerLength, err)
}
if start != nil {
updatedPerformer.CareerStart = models.NewOptionalInt(*start)
updatedPerformer.CareerStart = models.NewOptionalDate(*start)
}
if end != nil {
updatedPerformer.CareerEnd = models.NewOptionalInt(*end)
updatedPerformer.CareerEnd = models.NewOptionalDate(*end)
}
}
updatedPerformer.Tattoos = translator.optionalString(input.Tattoos, "tattoos")
@ -444,18 +459,24 @@ func (r *mutationResolver) BulkPerformerUpdate(ctx context.Context, input BulkPe
updatedPerformer.Circumcised = translator.optionalString((*string)(input.Circumcised), "circumcised")
// prefer career_start/career_end over deprecated career_length
if translator.hasField("career_start") || translator.hasField("career_end") {
updatedPerformer.CareerStart = translator.optionalInt(input.CareerStart, "career_start")
updatedPerformer.CareerEnd = translator.optionalInt(input.CareerEnd, "career_end")
updatedPerformer.CareerStart, err = translator.optionalDate(input.CareerStart, "career_start")
if err != nil {
return nil, fmt.Errorf("converting career start: %w", err)
}
updatedPerformer.CareerEnd, err = translator.optionalDate(input.CareerEnd, "career_end")
if err != nil {
return nil, fmt.Errorf("converting career end: %w", err)
}
} else if translator.hasField("career_length") && input.CareerLength != nil {
start, end, err := utils.ParseYearRangeString(*input.CareerLength)
start, end, err := models.ParseYearRangeString(*input.CareerLength)
if err != nil {
return nil, fmt.Errorf("could not parse career_length %q: %w", *input.CareerLength, err)
}
if start != nil {
updatedPerformer.CareerStart = models.NewOptionalInt(*start)
updatedPerformer.CareerStart = models.NewOptionalDate(*start)
}
if end != nil {
updatedPerformer.CareerEnd = models.NewOptionalInt(*end)
updatedPerformer.CareerEnd = models.NewOptionalDate(*end)
}
}
updatedPerformer.Tattoos = translator.optionalString(input.Tattoos, "tattoos")

17
internal/api/resolver_query_job.go
View file

@ -33,15 +33,26 @@ func (r *queryResolver) FindJob(ctx context.Context, input FindJobInput) (*Job,
}
func jobToJobModel(j job.Job) *Job {
subTasks := make([]string, len(j.Details))
for i, t := range j.Details {
subTasks[i] = sanitiseWebsocketString(t)
}
var jobError *string
if j.Error != nil {
s := sanitiseWebsocketString(*j.Error)
jobError = &s
}
ret := &Job{
ID: strconv.Itoa(j.ID),
Status: JobStatus(j.Status),
Description: j.Description,
SubTasks: j.Details,
Description: sanitiseWebsocketString(j.Description),
SubTasks: subTasks,
StartTime: j.StartTime,
EndTime: j.EndTime,
AddTime: j.AddTime,
Error: j.Error,
Error: jobError,
}
if j.Progress != -1 {

20
internal/api/resolver_query_scraper.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"slices"
"strconv"
"strings"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models"
@ -363,7 +364,8 @@ func (r *queryResolver) ScrapeSingleTag(ctx context.Context, source scraper.Sour
client := r.newStashBoxClient(*b)
var ret []*models.ScrapedTag
out, err := client.QueryTag(ctx, *input.Query)
query := *input.Query
out, err := client.QueryTag(ctx, query)
if err != nil {
return nil, err
@ -383,6 +385,22 @@ func (r *queryResolver) ScrapeSingleTag(ctx context.Context, source scraper.Sour
}); err != nil {
return nil, err
}
// tag name query returns results that may not match the query exactly.
// if there is an exact match, it should be first
if query != "" {
for i, result := range ret {
if strings.EqualFold(result.Name, query) {
// prepend exact match to the front of the slice
if i != 0 {
ret = append([]*models.ScrapedTag{result}, append(ret[:i], ret[i+1:]...)...)
}
break
}
}
}
return ret, nil
}

10
internal/api/resolver_subscription_logging.go
View file

@ -2,11 +2,19 @@ package api
import (
"context"
"strings"
"github.com/stashapp/stash/internal/log"
"github.com/stashapp/stash/internal/manager"
)
// sanitiseWebsocketString is used to ensure that any strings sent over the websocket are valid UTF-8.
// Any invalid UTF-8 sequences will be replaced with the Unicode replacement character (U+FFFD).
// Invalid UTF-8 sequences can cause the websocket connection to be closed.
func sanitiseWebsocketString(s string) string {
return strings.ToValidUTF8(s, "\uFFFD")
}
func getLogLevel(logType string) LogLevel {
switch logType {
case "progress":
@ -33,7 +41,7 @@ func logEntriesFromLogItems(logItems []log.LogItem) []*LogEntry {
ret[i] = &LogEntry{
Time: entry.Time,
Level: getLogLevel(entry.Type),
Message: entry.Message,
Message: sanitiseWebsocketString(entry.Message),
}
}

8
internal/manager/manager_tasks.go
View file

@ -314,6 +314,8 @@ type CleanMetadataInput struct {
Paths []string `json:"paths"`
// Do a dry run. Don't delete any files
DryRun bool `json:"dryRun"`
IgnoreZipFileContents bool `json:"ignoreZipFileContents"`
}
func (s *Manager) Clean(ctx context.Context, input CleanMetadataInput) int {
@ -431,7 +433,7 @@ type StashBoxBatchTagInput struct {
ExcludeFields []string `json:"exclude_fields"`
// Refresh items already tagged by StashBox if true. Only tag items with no StashBox tagging if false
Refresh bool `json:"refresh"`
// If batch adding studios, should their parent studios also be created?
// If batch adding studios or tags, should their parent entities also be created?
CreateParent bool `json:"createParent"`
// IDs in stash of the items to update.
// If set, names and stash_ids fields will be ignored.
@ -749,6 +751,7 @@ func (s *Manager) batchTagTagsByIds(ctx context.Context, input StashBoxBatchTagI
if (input.Refresh && hasStashID) || (!input.Refresh && !hasStashID) {
tasks = append(tasks, &stashBoxBatchTagTagTask{
tag: t,
createParent: input.CreateParent,
box: box,
excludedFields: input.ExcludeFields,
})
@ -769,6 +772,7 @@ func (s *Manager) batchTagTagsByNamesOrStashIds(input StashBoxBatchTagInput, box
if len(stashID) > 0 {
tasks = append(tasks, &stashBoxBatchTagTagTask{
stashID: &stashID,
createParent: input.CreateParent,
box: box,
excludedFields: input.ExcludeFields,
})
@ -780,6 +784,7 @@ func (s *Manager) batchTagTagsByNamesOrStashIds(input StashBoxBatchTagInput, box
if len(name) > 0 {
tasks = append(tasks, &stashBoxBatchTagTagTask{
name: &name,
createParent: input.CreateParent,
box: box,
excludedFields: input.ExcludeFields,
})
@ -806,6 +811,7 @@ func (s *Manager) batchTagAllTags(ctx context.Context, input StashBoxBatchTagInp
for _, t := range tags {
tasks = append(tasks, &stashBoxBatchTagTagTask{
tag: t,
createParent: input.CreateParent,
box: box,
excludedFields: input.ExcludeFields,
})

12
internal/manager/scan_stashignore_test.go
View file

@ -23,8 +23,8 @@ type stashIgnorePathFilter struct {
libraryRoot string
}
func (f *stashIgnorePathFilter) Accept(ctx context.Context, path string, info fs.FileInfo) bool {
return f.filter.Accept(ctx, path, info, f.libraryRoot)
func (f *stashIgnorePathFilter) Accept(ctx context.Context, path string, info fs.FileInfo, zipFilePath string) bool {
return f.filter.Accept(ctx, path, info, f.libraryRoot, zipFilePath)
}
// createTestFileOnDisk creates a file with some content.
@ -105,7 +105,7 @@ temp/
if err != nil {
t.Fatalf("failed to stat file %s: %v", scenario.path, err)
}
accepted := scanner.AcceptEntry(ctx, scenario.path, info)
accepted := scanner.AcceptEntry(ctx, scenario.path, info, "")
if accepted != scenario.accepted {
t.Errorf("unexpected accept result for %s: expected %v, got %v",
@ -160,7 +160,7 @@ func TestScannerWithNestedStashIgnore(t *testing.T) {
if err != nil {
t.Fatalf("failed to stat file %s: %v", scenario.path, err)
}
accepted := scanner.AcceptEntry(ctx, scenario.path, info)
accepted := scanner.AcceptEntry(ctx, scenario.path, info, "")
if accepted != scenario.accepted {
t.Errorf("unexpected accept result for %s: expected %v, got %v",
@ -205,7 +205,7 @@ func TestScannerWithoutStashIgnore(t *testing.T) {
if err != nil {
t.Fatalf("failed to stat file %s: %v", scenario.path, err)
}
accepted := scanner.AcceptEntry(ctx, scenario.path, info)
accepted := scanner.AcceptEntry(ctx, scenario.path, info, "")
if accepted != scenario.accepted {
t.Errorf("unexpected accept result for %s: expected %v, got %v",
@ -258,7 +258,7 @@ func TestScannerWithNegationPattern(t *testing.T) {
if err != nil {
t.Fatalf("failed to stat file %s: %v", scenario.path, err)
}
accepted := scanner.AcceptEntry(ctx, scenario.path, info)
accepted := scanner.AcceptEntry(ctx, scenario.path, info, "")
if accepted != scenario.accepted {
t.Errorf("unexpected accept result for %s: expected %v, got %v",

17
internal/manager/task/migrate.go
View file

@ -7,6 +7,8 @@ import (
"os"
"path/filepath"
"github.com/stashapp/stash/internal/manager/config"
"github.com/stashapp/stash/pkg/fsutil"
"github.com/stashapp/stash/pkg/job"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/sqlite"
@ -29,6 +31,21 @@ type databaseSchemaInfo struct {
StepsRequired uint
}
// PreExecute validates the environment before executing the migration.
// It returns an error if the migration cannot be performed.
func (s *MigrateJob) PreExecute() error {
// ensure backup directory exists and is writable
backupDir := s.Config.GetBackupDirectoryPathOrDefault()
if backupDir != "" {
if err := fsutil.EnsureDir(backupDir); err != nil {
logger.Errorf("error ensuring backup directory exists: %s", err)
logger.Warnf("Backup directory (%s) must be modified to a valid directory or removed from the config file", config.BackupDirectoryPath)
return fmt.Errorf("error creating backup directory: %w", err)
}
}
return nil
}
func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error {
schemaInfo, err := s.required()
if err != nil {

11
internal/manager/task_clean.go
View file

@ -40,9 +40,10 @@ func (j *cleanJob) Execute(ctx context.Context, progress *job.Progress) error {
}
j.cleaner.Clean(ctx, file.CleanOptions{
Paths: j.input.Paths,
DryRun: j.input.DryRun,
PathFilter: newCleanFilter(instance.Config),
Paths: j.input.Paths,
DryRun: j.input.DryRun,
IgnoreZipFileContents: j.input.IgnoreZipFileContents,
PathFilter: newCleanFilter(instance.Config),
}, progress)
if job.IsCancelled(ctx) {
@ -159,7 +160,7 @@ func newCleanFilter(c *config.Config) *cleanFilter {
}
}
func (f *cleanFilter) Accept(ctx context.Context, path string, info fs.FileInfo) bool {
func (f *cleanFilter) Accept(ctx context.Context, path string, info fs.FileInfo, zipFilePath string) bool {
// #1102 - clean anything in generated path
generatedPath := f.generatedPath
@ -184,7 +185,7 @@ func (f *cleanFilter) Accept(ctx context.Context, path string, info fs.FileInfo)
}
// Check .stashignore files, bounded to the library root.
if !f.stashIgnoreFilter.Accept(ctx, path, info, stash.Path) {
if !f.stashIgnoreFilter.Accept(ctx, path, info, stash.Path, zipFilePath) {
logger.Infof("%s is excluded due to .stashignore. Marking to clean: %q", fileOrFolder, path)
return false
}

29
internal/manager/task_scan.go
View file

@ -171,7 +171,12 @@ func (j *ScanJob) queueFileFunc(ctx context.Context, f models.FS, zipFile *file.
return nil
}
if !j.scanner.AcceptEntry(ctx, path, info) {
zipFilePath := ""
if zipFile != nil {
zipFilePath = zipFile.Path
}
if !j.scanner.AcceptEntry(ctx, path, info, zipFilePath) {
if info.IsDir() {
logger.Debugf("Skipping directory %s", path)
return fs.SkipDir
@ -278,8 +283,10 @@ func (j *ScanJob) processQueue(ctx context.Context, parallelTasks int, progress
for f := range j.fileQueue {
logger.Tracef("Processing queued file %s", f.Path)
if err := ctx.Err(); err != nil {
return
if ctx.Err() != nil {
// Keep receiving until queueFiles closes the channel; otherwise
// the walker can block on send (full buffer) and never finish.
continue
}
wg.Add()
@ -565,7 +572,7 @@ func newScanFilter(c *config.Config, repo models.Repository, minModTime time.Tim
}
}
func (f *scanFilter) Accept(ctx context.Context, path string, info fs.FileInfo) bool {
func (f *scanFilter) Accept(ctx context.Context, path string, info fs.FileInfo, zipFilePath string) bool {
if fsutil.IsPathInDir(f.generatedPath, path) {
logger.Warnf("Skipping %q as it overlaps with the generated folder", path)
return false
@ -583,7 +590,7 @@ func (f *scanFilter) Accept(ctx context.Context, path string, info fs.FileInfo)
}
// Check .stashignore files, bounded to the library root.
if !f.stashIgnoreFilter.Accept(ctx, path, info, s.Path) {
if !f.stashIgnoreFilter.Accept(ctx, path, info, s.Path, zipFilePath) {
logger.Debugf("Skipping %s due to .stashignore", path)
return false
}
@ -655,8 +662,9 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre
&file.FilteredHandler{
Filter: file.FilterFunc(imageFileFilter),
Handler: &image.ScanHandler{
CreatorUpdater: r.Image,
GalleryFinder: r.Gallery,
CreatorUpdater: r.Image,
GalleryFinder: r.Gallery,
SceneFinderUpdater: r.Scene,
ScanGenerator: &imageGenerators{
input: options,
taskQueue: taskQueue,
@ -685,9 +693,10 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre
&file.FilteredHandler{
Filter: file.FilterFunc(videoFileFilter),
Handler: &scene.ScanHandler{
CreatorUpdater: r.Scene,
CaptionUpdater: r.File,
PluginCache: pluginCache,
CreatorUpdater: r.Scene,
GalleryFinderUpdater: r.Gallery,
CaptionUpdater: r.File,
PluginCache: pluginCache,
ScanGenerator: &sceneGenerators{
input: options,
taskQueue: taskQueue,

65
internal/manager/task_stash_box_tag.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"strconv"
"strings"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/match"
@ -541,6 +542,7 @@ type stashBoxBatchTagTagTask struct {
name *string
stashID *string
tag *models.Tag
createParent bool
excludedFields []string
}
@ -588,8 +590,11 @@ func (t *stashBoxBatchTagTagTask) findStashBoxTag(ctx context.Context) (*models.
client := stashbox.NewClient(*t.box, stashbox.ExcludeTagPatterns(instance.Config.GetScraperExcludeTagPatterns()))
nameQuery := ""
switch {
case t.name != nil:
nameQuery = *t.name
results, err = client.QueryTag(ctx, *t.name)
case t.stashID != nil:
results, err = client.QueryTag(ctx, *t.stashID)
@ -615,6 +620,7 @@ func (t *stashBoxBatchTagTagTask) findStashBoxTag(ctx context.Context) (*models.
if remoteID != "" {
results, err = client.QueryTag(ctx, remoteID)
} else {
nameQuery = t.tag.Name
results, err = client.QueryTag(ctx, t.tag.Name)
}
}
@ -627,10 +633,26 @@ func (t *stashBoxBatchTagTagTask) findStashBoxTag(ctx context.Context) (*models.
return nil, nil
}
result := results[0]
var result *models.ScrapedTag
// QueryTag returns tags that partially match the name, so find the exact match if searching by name
if nameQuery != "" {
for _, r := range results {
if strings.EqualFold(r.Name, nameQuery) {
result = r
break
}
}
} else {
result = results[0]
}
if result == nil {
return nil, nil
}
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
return match.ScrapedTag(ctx, r.Tag, result, t.box.Endpoint)
return match.ScrapedTagHierarchy(ctx, r.Tag, result, t.box.Endpoint)
}); err != nil {
return nil, err
}
@ -638,6 +660,39 @@ func (t *stashBoxBatchTagTagTask) findStashBoxTag(ctx context.Context) (*models.
return result, nil
}
func (t *stashBoxBatchTagTagTask) processParentTag(ctx context.Context, parent *models.ScrapedTag, excluded map[string]bool) error {
if parent.StoredID == nil {
// Create new parent tag
newParentTag := parent.ToTag(t.box.Endpoint, excluded)
r := instance.Repository
err := r.WithTxn(ctx, func(ctx context.Context) error {
qb := r.Tag
if err := tag.ValidateCreate(ctx, *newParentTag, qb); err != nil {
return err
}
if err := qb.Create(ctx, &models.CreateTagInput{Tag: newParentTag}); err != nil {
return err
}
storedID := strconv.Itoa(newParentTag.ID)
parent.StoredID = &storedID
return nil
})
if err != nil {
logger.Errorf("Failed to create parent tag %s: %v", parent.Name, err)
} else {
logger.Infof("Created parent tag %s", parent.Name)
}
return err
}
// Parent already exists — nothing to update for categories
return nil
}
func (t *stashBoxBatchTagTagTask) processMatchedTag(ctx context.Context, s *models.ScrapedTag, excluded map[string]bool) {
// Determine the tag ID to update — either from the task's tag or from the
// StoredID set by match.ScrapedTag (when batch adding by name and the tag
@ -649,6 +704,12 @@ func (t *stashBoxBatchTagTagTask) processMatchedTag(ctx context.Context, s *mode
tagID, _ = strconv.Atoi(*s.StoredID)
}
if s.Parent != nil && t.createParent {
if err := t.processParentTag(ctx, s.Parent, excluded); err != nil {
return
}
}
if tagID > 0 {
r := instance.Repository
err := r.WithTxn(ctx, func(ctx context.Context) error {

8
pkg/ffmpeg/codec_hardware.go
View file

@ -185,6 +185,12 @@ func (f *FFMpeg) hwCanFullHWTranscode(ctx context.Context, codec VideoCodec, vf
// Prepend input for hardware encoding only
func (f *FFMpeg) hwDeviceInit(args Args, toCodec VideoCodec, fullhw bool) Args {
// check for custom /dev/dri device #6435
driDevice := os.Getenv("STASH_HW_DRI_DEVICE")
if driDevice == "" {
driDevice = "/dev/dri/renderD128"
}
switch toCodec {
case VideoCodecN264,
VideoCodecN264H:
@ -201,7 +207,7 @@ func (f *FFMpeg) hwDeviceInit(args Args, toCodec VideoCodec, fullhw bool) Args {
case VideoCodecV264,
VideoCodecVVP9:
args = append(args, "-vaapi_device")
args = append(args, "/dev/dri/renderD128")
args = append(args, driDevice)
if fullhw {
args = append(args, "-hwaccel")
args = append(args, "vaapi")

30
pkg/file/clean.go
View file

@ -33,6 +33,11 @@ type cleanJob struct {
type CleanOptions struct {
Paths []string
// IgnoreZipFileContents will skip checking the contents of zip files when determining whether to clean a file.
// This can significantly speed up the clean process, but will potentially miss removed files within zip files.
// Where users do not modify zip files contents directly, this should be safe to use.
IgnoreZipFileContents bool
// Do a dry run. Don't delete any files
DryRun bool
@ -174,13 +179,16 @@ func (j *cleanJob) assessFiles(ctx context.Context, toDelete *deleteSet) error {
more := true
r := j.Repository
includeZipContents := !j.options.IgnoreZipFileContents
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
for more {
if job.IsCancelled(ctx) {
return nil
}
files, err := r.File.FindAllInPaths(ctx, j.options.Paths, batchSize, offset)
files, err := r.File.FindAllInPaths(ctx, j.options.Paths, includeZipContents, batchSize, offset)
if err != nil {
return fmt.Errorf("error querying for files: %w", err)
}
@ -258,6 +266,8 @@ func (j *cleanJob) assessFolders(ctx context.Context, toDelete *deleteSet) error
offset := 0
progress := j.progress
includeZipContents := !j.options.IgnoreZipFileContents
more := true
r := j.Repository
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
@ -266,7 +276,7 @@ func (j *cleanJob) assessFolders(ctx context.Context, toDelete *deleteSet) error
return nil
}
folders, err := r.Folder.FindAllInPaths(ctx, j.options.Paths, batchSize, offset)
folders, err := r.Folder.FindAllInPaths(ctx, j.options.Paths, includeZipContents, batchSize, offset)
if err != nil {
return fmt.Errorf("error querying for folders: %w", err)
}
@ -348,8 +358,14 @@ func (j *cleanJob) shouldClean(ctx context.Context, f models.File) bool {
// run through path filter, if returns false then the file should be cleaned
filter := j.options.PathFilter
// need to get the zip file path if present
zipFilePath := ""
if f.Base().ZipFile != nil {
zipFilePath = f.Base().ZipFile.Base().Path
}
// don't log anything - assume filter will have logged the reason
return !filter.Accept(ctx, path, info)
return !filter.Accept(ctx, path, info, zipFilePath)
}
func (j *cleanJob) shouldCleanFolder(ctx context.Context, f *models.Folder) bool {
@ -387,8 +403,14 @@ func (j *cleanJob) shouldCleanFolder(ctx context.Context, f *models.Folder) bool
// run through path filter, if returns false then the file should be cleaned
filter := j.options.PathFilter
// need to get the zip file path if present
zipFilePath := ""
if f.ZipFile != nil {
zipFilePath = f.ZipFile.Base().Path
}
// don't log anything - assume filter will have logged the reason
return !filter.Accept(ctx, path, info)
return !filter.Accept(ctx, path, info, zipFilePath)
}
func (j *cleanJob) deleteFile(ctx context.Context, fileID models.FileID, fn string) {

12
pkg/file/folder_rename_detect.go
View file

@ -2,7 +2,6 @@ package file
import (
"context"
"errors"
"fmt"
"io/fs"
@ -88,6 +87,11 @@ func (s *Scanner) detectFolderMove(ctx context.Context, file ScannedFile) (*mode
r := s.Repository
zipFilePath := ""
if file.ZipFile != nil {
zipFilePath = file.ZipFile.Base().Path
}
if err := SymWalk(file.FS, file.Path, func(path string, d fs.DirEntry, err error) error {
if err != nil {
// don't let errors prevent scanning
@ -111,7 +115,7 @@ func (s *Scanner) detectFolderMove(ctx context.Context, file ScannedFile) (*mode
return nil
}
if !s.AcceptEntry(ctx, path, info) {
if !s.AcceptEntry(ctx, path, info, zipFilePath) {
return nil
}
@ -161,9 +165,7 @@ func (s *Scanner) detectFolderMove(ctx context.Context, file ScannedFile) (*mode
continue
}
if !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("checking for parent folder %q: %w", pf.Path, err)
}
// treat any error as missing folder
// parent folder is missing, possible candidate
// count the total number of files in the existing folder

2
pkg/file/handler.go
View file

@ -9,7 +9,7 @@ import (
// PathFilter provides a filter function for paths.
type PathFilter interface {
Accept(ctx context.Context, path string, info fs.FileInfo) bool
Accept(ctx context.Context, path string, info fs.FileInfo, zipFilePath string) bool
}
type PathFilterFunc func(path string) bool

14
pkg/file/scan.go
View file

@ -111,12 +111,12 @@ type ScannedFile struct {
}
// AcceptEntry determines if the file entry should be accepted for scanning
func (s *Scanner) AcceptEntry(ctx context.Context, path string, info fs.FileInfo) bool {
func (s *Scanner) AcceptEntry(ctx context.Context, path string, info fs.FileInfo, zipFilePath string) bool {
// always accept if there's no filters
accept := len(s.ScanFilters) == 0
for _, filter := range s.ScanFilters {
// accept if any filter accepts the file
if filter.Accept(ctx, path, info) {
if filter.Accept(ctx, path, info, zipFilePath) {
accept = true
break
}
@ -462,7 +462,11 @@ func (s *Scanner) onNewFile(ctx context.Context, f ScannedFile) (*ScanFileResult
// determine if the file is renamed from an existing file in the store
// do this after decoration so that missing fields can be populated
renamed, err := s.handleRename(ctx, file, fp)
zipFilePath := ""
if f.ZipFile != nil {
zipFilePath = f.ZipFile.Base().Path
}
renamed, err := s.handleRename(ctx, file, fp, zipFilePath)
if err != nil {
return nil, err
}
@ -572,7 +576,7 @@ func (s *Scanner) getFileFS(f *models.BaseFile) (models.FS, error) {
return fs.OpenZip(zipPath, zipSize)
}
func (s *Scanner) handleRename(ctx context.Context, f models.File, fp []models.Fingerprint) (models.File, error) {
func (s *Scanner) handleRename(ctx context.Context, f models.File, fp []models.Fingerprint, zipFilePath string) (models.File, error) {
var others []models.File
for _, tfp := range fp {
@ -614,7 +618,7 @@ func (s *Scanner) handleRename(ctx context.Context, f models.File, fp []models.F
// treat as a move
missing = append(missing, other)
}
case !s.AcceptEntry(ctx, other.Base().Path, info):
case !s.AcceptEntry(ctx, other.Base().Path, info, zipFilePath):
// #4393 - if the file is no longer in the configured library paths, treat it as a move
logger.Debugf("File %q no longer in library paths. Treating as a move.", other.Base().Path)
missing = append(missing, other)

9
pkg/file/stashignore.go
View file

@ -53,7 +53,9 @@ func NewStashIgnoreFilter() *StashIgnoreFilter {
// applies gitignore-style pattern matching.
// The libraryRoot parameter bounds the search for .stashignore files -
// only directories within the library root are checked.
func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.FileInfo, libraryRoot string) bool {
// zipFilepath is the path of the zip file if the file is inside a zip.
// .stashignore files will not be read within zip files.
func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.FileInfo, libraryRoot string, zipFilePath string) bool {
// If no library root provided, accept the file (safety fallback).
if libraryRoot == "" {
return true
@ -62,6 +64,11 @@ func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.Fil
// Get the directory containing this path.
dir := filepath.Dir(path)
// If the file is inside a zip, use the zip file's directory as the base for .stashignore lookup.
if zipFilePath != "" {
dir = filepath.Dir(zipFilePath)
}
// Collect all applicable ignore entries from library root to this directory.
entries := f.collectIgnoreEntries(dir, libraryRoot)

2
pkg/file/stashignore_test.go
View file

@ -64,7 +64,7 @@ func walkAndFilter(t *testing.T, root string, filter *StashIgnoreFilter) []strin
return err
}
if filter.Accept(ctx, path, info, root) {
if filter.Accept(ctx, path, info, root, "") {
relPath, _ := filepath.Rel(root, path)
accepted = append(accepted, relPath)
} else if info.IsDir() {

2
pkg/fsutil/file.go
View file

@ -148,7 +148,7 @@ func Touch(path string) error {
var (
replaceCharsRE = regexp.MustCompile(`[&=\\/:*"?_ ]`)
removeCharsRE = regexp.MustCompile(`[^[:alnum:]-.]`)
removeCharsRE = regexp.MustCompile(`[^\p{L}\p{N}\-.]`)
multiHyphenRE = regexp.MustCompile(`\-+`)
)

3
pkg/fsutil/file_test.go
View file

@ -15,6 +15,9 @@ func TestSanitiseBasename(t *testing.T) {
{"multi-hyphen", `hyphened--name`, "hyphened-name-2da2a58f"},
{"replaced characters", `a&b=c\d/:e*"f?_ g`, "a-b-c-d-e-f-g-ffca6fb0"},
{"removed characters", `foo!!bar@@and, more`, "foobarand-more-7cee02ab"},
{"unicode cjk", `テスト`, "テスト-63b560db"},
{"unicode korean", `시험`, "시험-3fcc7beb"},
{"mixed unicode", `Test テスト`, "Test-テスト-366aff1e"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

1
pkg/gallery/scan.go
View file

@ -24,7 +24,6 @@ type ScanCreatorUpdater interface {
type ScanSceneFinderUpdater interface {
FindByPath(ctx context.Context, p string) ([]*models.Scene, error)
Update(ctx context.Context, updatedScene *models.Scene) error
AddGalleryIDs(ctx context.Context, sceneID int, galleryIDs []int) error
}

39
pkg/image/scan.go
View file

@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"slices"
"strings"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
@ -39,6 +40,11 @@ type GalleryFinderCreator interface {
UpdatePartial(ctx context.Context, id int, updatedGallery models.GalleryPartial) (*models.Gallery, error)
}
type ScanSceneFinderUpdater interface {
FindByPath(ctx context.Context, p string) ([]*models.Scene, error)
AddGalleryIDs(ctx context.Context, sceneID int, galleryIDs []int) error
}
type ScanConfig interface {
GetCreateGalleriesFromFolders() bool
}
@ -48,8 +54,9 @@ type ScanGenerator interface {
}
type ScanHandler struct {
CreatorUpdater ScanCreatorUpdater
GalleryFinder GalleryFinderCreator
CreatorUpdater ScanCreatorUpdater
GalleryFinder GalleryFinderCreator
SceneFinderUpdater ScanSceneFinderUpdater
ScanGenerator ScanGenerator
@ -322,11 +329,39 @@ func (h *ScanHandler) getOrCreateZipBasedGallery(ctx context.Context, zipFile mo
return nil, fmt.Errorf("creating zip-based gallery: %w", err)
}
// try to associate with scene
if err := h.associateScene(ctx, &newGallery, zipFile); err != nil {
return nil, fmt.Errorf("associating scene: %w", err)
}
h.PluginCache.RegisterPostHooks(ctx, newGallery.ID, hook.GalleryCreatePost, nil, nil)
return &newGallery, nil
}
func (h *ScanHandler) associateScene(ctx context.Context, existing *models.Gallery, zipFile models.File) error {
galleryIDs := []int{existing.ID}
path := zipFile.Base().Path
withoutExt := strings.TrimSuffix(path, filepath.Ext(path)) + ".*"
// find scenes with a file that matches
scenes, err := h.SceneFinderUpdater.FindByPath(ctx, withoutExt)
if err != nil {
return err
}
for _, scene := range scenes {
// found related Scene
logger.Infof("associate: Gallery %s is related to scene: %d", path, scene.ID)
if err := h.SceneFinderUpdater.AddGalleryIDs(ctx, scene.ID, galleryIDs); err != nil {
return err
}
}
return nil
}
func (h *ScanHandler) getOrCreateGallery(ctx context.Context, f models.File) (*models.Gallery, error) {
// don't create folder-based galleries for files in zip file
if f.Base().ZipFile != nil {

17
pkg/job/job.go
View file

@ -66,6 +66,23 @@ type Job struct {
cancelFunc context.CancelFunc
}
// statusCopy returns a copy of the Job with only the fields needed for
// status reporting. Internal fields (exec, cancelFunc, outerCtx) are
// excluded so that subscription channels don't retain heavy resources.
func (j *Job) statusCopy() Job {
return Job{
ID: j.ID,
Status: j.Status,
Details: j.Details,
Description: j.Description,
Progress: j.Progress,
StartTime: j.StartTime,
EndTime: j.EndTime,
AddTime: j.AddTime,
Error: j.Error,
}
}
// TimeElapsed returns the total time elapsed for the job.
// If the EndTime is set, then it uses this to calculate the elapsed time, otherwise it uses time.Now.
func (j *Job) TimeElapsed() time.Duration {

16
pkg/job/manager.go
View file

@ -105,7 +105,7 @@ func (m *Manager) notifyNewJob(j *Job) {
for _, s := range m.subscriptions {
// don't block if channel is full
select {
case s.newJob <- *j:
case s.newJob <- j.statusCopy():
default:
}
}
@ -232,7 +232,9 @@ func (m *Manager) removeJob(job *Job) {
return
}
// clear any subtasks
// release the executor and subtask details so they can be GC'd
// while the job remains in the graveyard for status reporting
job.exec = nil
job.Details = nil
m.queue = append(m.queue[:index], m.queue[index+1:]...)
@ -246,7 +248,7 @@ func (m *Manager) removeJob(job *Job) {
for _, s := range m.subscriptions {
// don't block if channel is full
select {
case s.removedJob <- *job:
case s.removedJob <- job.statusCopy():
default:
}
}
@ -310,8 +312,7 @@ func (m *Manager) GetJob(id int) *Job {
// get from the queue or graveyard
_, j := m.getJob(append(m.queue, m.graveyard...), id)
if j != nil {
// make a copy of the job and return the pointer
jCopy := *j
jCopy := j.statusCopy()
return &jCopy
}
@ -326,8 +327,7 @@ func (m *Manager) GetQueue() []Job {
var ret []Job
for _, j := range m.queue {
jCopy := *j
ret = append(ret, jCopy)
ret = append(ret, j.statusCopy())
}
return ret
@ -372,7 +372,7 @@ func (m *Manager) notifyJobUpdate(j *Job) {
for _, s := range m.subscriptions {
// don't block if channel is full
select {
case s.updatedJob <- *j:
case s.updatedJob <- j.statusCopy():
default:
}
}

2
pkg/job/task.go
View file

@ -51,7 +51,7 @@ func (tq *TaskQueue) executer(ctx context.Context) {
defer tq.wg.Wait()
for task := range tq.tasks {
if IsCancelled(ctx) {
return
continue // allow channel to continue draining until Close()
}
tt := task

18
pkg/match/scraped.go
View file

@ -22,7 +22,7 @@ type GroupNamesFinder interface {
type SceneRelationships struct {
PerformerFinder PerformerFinder
TagFinder models.TagQueryer
TagFinder models.TagNameFinder
StudioFinder StudioFinder
}
@ -188,9 +188,23 @@ func ScrapedGroup(ctx context.Context, qb GroupNamesFinder, storedID *string, na
return
}
// ScrapedTagHierarchy executes ScrapedTag for the provided tag and its parent.
func ScrapedTagHierarchy(ctx context.Context, qb models.TagNameFinder, s *models.ScrapedTag, stashBoxEndpoint string) error {
if err := ScrapedTag(ctx, qb, s, stashBoxEndpoint); err != nil {
return err
}
if s.Parent == nil {
return nil
}
// Match parent by name only (categories don't have StashDB tag IDs)
return ScrapedTag(ctx, qb, s.Parent, "")
}
// ScrapedTag matches the provided tag with the tags
// in the database and sets the ID field if one is found.
func ScrapedTag(ctx context.Context, qb models.TagQueryer, s *models.ScrapedTag, stashBoxEndpoint string) error {
func ScrapedTag(ctx context.Context, qb models.TagNameFinder, s *models.ScrapedTag, stashBoxEndpoint string) error {
if s.StoredID != nil {
return nil
}

112
pkg/models/date.go
View file

@ -2,6 +2,7 @@ package models
import (
"fmt"
"strings"
"time"
"github.com/stashapp/stash/pkg/utils"
@ -61,3 +62,114 @@ func ParseDate(s string) (Date, error) {
return Date{}, fmt.Errorf("failed to parse date %q: %v", s, errs)
}
func DateFromYear(year int) Date {
return Date{
Time: time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC),
Precision: DatePrecisionYear,
}
}
func FormatYearRange(start *Date, end *Date) string {
var (
startStr, endStr string
)
if start != nil {
startStr = start.Format(dateFormatPrecision[DatePrecisionYear])
}
if end != nil {
endStr = end.Format(dateFormatPrecision[DatePrecisionYear])
}
switch {
case startStr == "" && endStr == "":
return ""
case endStr == "":
return fmt.Sprintf("%s -", startStr)
case startStr == "":
return fmt.Sprintf("- %s", endStr)
default:
return fmt.Sprintf("%s - %s", startStr, endStr)
}
}
func FormatYearRangeString(start *string, end *string) string {
switch {
case start == nil && end == nil:
return ""
case end == nil:
return fmt.Sprintf("%s -", *start)
case start == nil:
return fmt.Sprintf("- %s", *end)
default:
return fmt.Sprintf("%s - %s", *start, *end)
}
}
// ParseYearRangeString parses a year range string into start and end year integers.
// Supported formats: "YYYY", "YYYY - YYYY", "YYYY-YYYY", "YYYY -", "- YYYY", "YYYY-present".
// Returns nil for start/end if not present in the string.
func ParseYearRangeString(s string) (start *Date, end *Date, err error) {
s = strings.TrimSpace(s)
if s == "" {
return nil, nil, fmt.Errorf("empty year range string")
}
// normalize "present" to empty end
lower := strings.ToLower(s)
lower = strings.ReplaceAll(lower, "present", "")
// split on "-" if it contains one
var parts []string
if strings.Contains(lower, "-") {
parts = strings.SplitN(lower, "-", 2)
} else {
// single value, treat as start year
year, err := parseYear(lower)
if err != nil {
return nil, nil, fmt.Errorf("invalid year range %q: %w", s, err)
}
return year, nil, nil
}
startStr := strings.TrimSpace(parts[0])
endStr := strings.TrimSpace(parts[1])
if startStr != "" {
y, err := parseYear(startStr)
if err != nil {
return nil, nil, fmt.Errorf("invalid start year in %q: %w", s, err)
}
start = y
}
if endStr != "" {
y, err := parseYear(endStr)
if err != nil {
return nil, nil, fmt.Errorf("invalid end year in %q: %w", s, err)
}
end = y
}
if start == nil && end == nil {
return nil, nil, fmt.Errorf("could not parse year range %q", s)
}
return start, end, nil
}
func parseYear(s string) (*Date, error) {
ret, err := ParseDate(s)
if err != nil {
return nil, fmt.Errorf("parsing year %q: %w", s, err)
}
year := ret.Time.Year()
if year < 1900 || year > 2200 {
return nil, fmt.Errorf("year %d out of reasonable range", year)
}
return &ret, nil
}

101
pkg/models/date_test.go
View file

@ -3,6 +3,8 @@ package models
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestParseDateStringAsTime(t *testing.T) {
@ -48,3 +50,102 @@ func TestParseDateStringAsTime(t *testing.T) {
})
}
}
func TestFormatYearRange(t *testing.T) {
datePtr := func(v int) *Date {
date := DateFromYear(v)
return &date
}
tests := []struct {
name string
start *Date
end *Date
want string
}{
{"both nil", nil, nil, ""},
{"only start", datePtr(2005), nil, "2005 -"},
{"only end", nil, datePtr(2010), "- 2010"},
{"start and end", datePtr(2005), datePtr(2010), "2005 - 2010"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := FormatYearRange(tt.start, tt.end)
assert.Equal(t, tt.want, got)
})
}
}
func TestFormatYearRangeString(t *testing.T) {
stringPtr := func(v string) *string { return &v }
tests := []struct {
name string
start *string
end *string
want string
}{
{"both nil", nil, nil, ""},
{"only start", stringPtr("2005"), nil, "2005 -"},
{"only end", nil, stringPtr("2010"), "- 2010"},
{"start and end", stringPtr("2005"), stringPtr("2010"), "2005 - 2010"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := FormatYearRangeString(tt.start, tt.end)
assert.Equal(t, tt.want, got)
})
}
}
func TestParseYearRangeString(t *testing.T) {
intPtr := func(v int) *int { return &v }
tests := []struct {
name string
input string
wantStart *int
wantEnd *int
wantErr bool
}{
{"single year", "2005", intPtr(2005), nil, false},
{"year range with spaces", "2005 - 2010", intPtr(2005), intPtr(2010), false},
{"year range no spaces", "2005-2010", intPtr(2005), intPtr(2010), false},
{"year dash open", "2005 -", intPtr(2005), nil, false},
{"year dash open no space", "2005-", intPtr(2005), nil, false},
{"dash year", "- 2010", nil, intPtr(2010), false},
{"year present", "2005-present", intPtr(2005), nil, false},
{"year Present caps", "2005 - Present", intPtr(2005), nil, false},
{"whitespace padding", " 2005 - 2010 ", intPtr(2005), intPtr(2010), false},
{"empty string", "", nil, nil, true},
{"garbage", "not a year", nil, nil, true},
{"partial garbage start", "abc - 2010", nil, nil, true},
{"partial garbage end", "2005 - abc", nil, nil, true},
{"year out of range", "1800", nil, nil, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
start, end, err := ParseYearRangeString(tt.input)
if tt.wantErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
if tt.wantStart != nil {
assert.NotNil(t, start)
assert.Equal(t, *tt.wantStart, start.Time.Year())
} else {
assert.Nil(t, start)
}
if tt.wantEnd != nil {
assert.NotNil(t, end)
assert.Equal(t, *tt.wantEnd, end.Time.Year())
} else {
assert.Nil(t, end)
}
})
}
}

4
pkg/models/jsonschema/performer.go
View file

@ -49,8 +49,8 @@ type Performer struct {
PenisLength float64 `json:"penis_length,omitempty"`
Circumcised string `json:"circumcised,omitempty"`
CareerLength string `json:"career_length,omitempty"` // deprecated - for import only
CareerStart *int `json:"career_start,omitempty"`
CareerEnd *int `json:"career_end,omitempty"`
CareerStart string `json:"career_start,omitempty"`
CareerEnd string `json:"career_end,omitempty"`
Tattoos string `json:"tattoos,omitempty"`
Piercings string `json:"piercings,omitempty"`
Aliases StringOrStringList `json:"aliases,omitempty"`

14
pkg/models/mocks/FileReaderWriter.go
View file

@ -153,13 +153,13 @@ func (_m *FileReaderWriter) FindAllByPath(ctx context.Context, path string, case
return r0, r1
}
// FindAllInPaths provides a mock function with given fields: ctx, p, limit, offset
func (_m *FileReaderWriter) FindAllInPaths(ctx context.Context, p []string, limit int, offset int) ([]models.File, error) {
ret := _m.Called(ctx, p, limit, offset)
// FindAllInPaths provides a mock function with given fields: ctx, p, includeZipContents, limit, offset
func (_m *FileReaderWriter) FindAllInPaths(ctx context.Context, p []string, includeZipContents bool, limit int, offset int) ([]models.File, error) {
ret := _m.Called(ctx, p, includeZipContents, limit, offset)
var r0 []models.File
if rf, ok := ret.Get(0).(func(context.Context, []string, int, int) []models.File); ok {
r0 = rf(ctx, p, limit, offset)
if rf, ok := ret.Get(0).(func(context.Context, []string, bool, int, int) []models.File); ok {
r0 = rf(ctx, p, includeZipContents, limit, offset)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]models.File)
@ -167,8 +167,8 @@ func (_m *FileReaderWriter) FindAllInPaths(ctx context.Context, p []string, limi
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, []string, int, int) error); ok {
r1 = rf(ctx, p, limit, offset)
if rf, ok := ret.Get(1).(func(context.Context, []string, bool, int, int) error); ok {
r1 = rf(ctx, p, includeZipContents, limit, offset)
} else {
r1 = ret.Error(1)
}

37
pkg/models/mocks/FolderReaderWriter.go
View file

@ -86,13 +86,13 @@ func (_m *FolderReaderWriter) Find(ctx context.Context, id models.FolderID) (*mo
return r0, r1
}
// FindAllInPaths provides a mock function with given fields: ctx, p, limit, offset
func (_m *FolderReaderWriter) FindAllInPaths(ctx context.Context, p []string, limit int, offset int) ([]*models.Folder, error) {
ret := _m.Called(ctx, p, limit, offset)
// FindAllInPaths provides a mock function with given fields: ctx, p, includeZipContents, limit, offset
func (_m *FolderReaderWriter) FindAllInPaths(ctx context.Context, p []string, includeZipContents bool, limit int, offset int) ([]*models.Folder, error) {
ret := _m.Called(ctx, p, includeZipContents, limit, offset)
var r0 []*models.Folder
if rf, ok := ret.Get(0).(func(context.Context, []string, int, int) []*models.Folder); ok {
r0 = rf(ctx, p, limit, offset)
if rf, ok := ret.Get(0).(func(context.Context, []string, bool, int, int) []*models.Folder); ok {
r0 = rf(ctx, p, includeZipContents, limit, offset)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*models.Folder)
@ -100,8 +100,8 @@ func (_m *FolderReaderWriter) FindAllInPaths(ctx context.Context, p []string, li
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, []string, int, int) error); ok {
r1 = rf(ctx, p, limit, offset)
if rf, ok := ret.Get(1).(func(context.Context, []string, bool, int, int) error); ok {
r1 = rf(ctx, p, includeZipContents, limit, offset)
} else {
r1 = ret.Error(1)
}
@ -224,6 +224,29 @@ func (_m *FolderReaderWriter) GetManyParentFolderIDs(ctx context.Context, folder
return r0, r1
}
// GetManySubFolderIDs provides a mock function with given fields: ctx, folderIDs
func (_m *FolderReaderWriter) GetManySubFolderIDs(ctx context.Context, folderIDs []models.FolderID) ([][]models.FolderID, error) {
ret := _m.Called(ctx, folderIDs)
var r0 [][]models.FolderID
if rf, ok := ret.Get(0).(func(context.Context, []models.FolderID) [][]models.FolderID); ok {
r0 = rf(ctx, folderIDs)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([][]models.FolderID)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, []models.FolderID) error); ok {
r1 = rf(ctx, folderIDs)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Query provides a mock function with given fields: ctx, options
func (_m *FolderReaderWriter) Query(ctx context.Context, options models.FolderQueryOptions) (*models.FolderQueryResult, error) {
ret := _m.Called(ctx, options)

14
pkg/models/mocks/GalleryReaderWriter.go
View file

@ -49,6 +49,20 @@ func (_m *GalleryReaderWriter) AddImages(ctx context.Context, galleryID int, ima
return r0
}
// AddSceneIDs provides a mock function with given fields: ctx, galleryID, sceneIDs
func (_m *GalleryReaderWriter) AddSceneIDs(ctx context.Context, galleryID int, sceneIDs []int) error {
ret := _m.Called(ctx, galleryID, sceneIDs)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, int, []int) error); ok {
r0 = rf(ctx, galleryID, sceneIDs)
} else {
r0 = ret.Error(0)
}
return r0
}
// All provides a mock function with given fields: ctx
func (_m *GalleryReaderWriter) All(ctx context.Context) ([]*models.Gallery, error) {
ret := _m.Called(ctx)

23
pkg/models/mocks/TagReaderWriter.go
View file

@ -197,6 +197,29 @@ func (_m *TagReaderWriter) FindAllDescendants(ctx context.Context, tagID int, ex
return r0, r1
}
// FindByAlias provides a mock function with given fields: ctx, alias, nocase
func (_m *TagReaderWriter) FindByAlias(ctx context.Context, alias string, nocase bool) (*models.Tag, error) {
ret := _m.Called(ctx, alias, nocase)
var r0 *models.Tag
if rf, ok := ret.Get(0).(func(context.Context, string, bool) *models.Tag); ok {
r0 = rf(ctx, alias, nocase)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*models.Tag)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok {
r1 = rf(ctx, alias, nocase)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FindByChildTagID provides a mock function with given fields: ctx, childID
func (_m *TagReaderWriter) FindByChildTagID(ctx context.Context, childID int) ([]*models.Tag, error) {
ret := _m.Called(ctx, childID)

44
pkg/models/model_performer.go
View file

@ -6,26 +6,26 @@ import (
)
type Performer struct {
ID int `json:"id"`
Name string `json:"name"`
Disambiguation string `json:"disambiguation"`
Gender *GenderEnum `json:"gender"`
Birthdate *Date `json:"birthdate"`
Ethnicity string `json:"ethnicity"`
Country string `json:"country"`
EyeColor string `json:"eye_color"`
Height *int `json:"height"`
Measurements string `json:"measurements"`
FakeTits string `json:"fake_tits"`
PenisLength *float64 `json:"penis_length"`
Circumcised *CircumisedEnum `json:"circumcised"`
CareerStart *int `json:"career_start"`
CareerEnd *int `json:"career_end"`
Tattoos string `json:"tattoos"`
Piercings string `json:"piercings"`
Favorite bool `json:"favorite"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ID int `json:"id"`
Name string `json:"name"`
Disambiguation string `json:"disambiguation"`
Gender *GenderEnum `json:"gender"`
Birthdate *Date `json:"birthdate"`
Ethnicity string `json:"ethnicity"`
Country string `json:"country"`
EyeColor string `json:"eye_color"`
Height *int `json:"height"`
Measurements string `json:"measurements"`
FakeTits string `json:"fake_tits"`
PenisLength *float64 `json:"penis_length"`
Circumcised *CircumcisedEnum `json:"circumcised"`
CareerStart *Date `json:"career_start"`
CareerEnd *Date `json:"career_end"`
Tattoos string `json:"tattoos"`
Piercings string `json:"piercings"`
Favorite bool `json:"favorite"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
// Rating expressed in 1-100 scale
Rating *int `json:"rating"`
Details string `json:"details"`
@ -76,8 +76,8 @@ type PerformerPartial struct {
FakeTits OptionalString
PenisLength OptionalFloat64
Circumcised OptionalString
CareerStart OptionalInt
CareerEnd OptionalInt
CareerStart OptionalDate
CareerEnd OptionalDate
Tattoos OptionalString
Piercings OptionalString
Favorite OptionalBool

52
pkg/models/model_scraped_item.go
View file

@ -177,8 +177,8 @@ type ScrapedPerformer struct {
PenisLength *string `json:"penis_length"`
Circumcised *string `json:"circumcised"`
CareerLength *string `json:"career_length"` // deprecated: use CareerStart/CareerEnd
CareerStart *int `json:"career_start"`
CareerEnd *int `json:"career_end"`
CareerStart *string `json:"career_start"`
CareerEnd *string `json:"career_end"`
Tattoos *string `json:"tattoos"`
Piercings *string `json:"piercings"`
Aliases *string `json:"aliases"`
@ -225,12 +225,16 @@ func (p *ScrapedPerformer) ToPerformer(endpoint string, excluded map[string]bool
// assume that career length is _not_ populated in favour of start/end
if p.CareerStart != nil && !excluded["career_start"] {
cs := *p.CareerStart
ret.CareerStart = &cs
date, err := ParseDate(*p.CareerStart)
if err == nil {
ret.CareerStart = &date
}
}
if p.CareerEnd != nil && !excluded["career_end"] {
ce := *p.CareerEnd
ret.CareerEnd = &ce
date, err := ParseDate(*p.CareerEnd)
if err == nil {
ret.CareerEnd = &date
}
}
if p.Country != nil && !excluded["country"] {
ret.Country = *p.Country
@ -288,7 +292,7 @@ func (p *ScrapedPerformer) ToPerformer(endpoint string, excluded map[string]bool
}
}
if p.Circumcised != nil && !excluded["circumcised"] {
v := CircumisedEnum(*p.Circumcised)
v := CircumcisedEnum(*p.Circumcised)
if v.IsValid() {
ret.Circumcised = &v
}
@ -367,13 +371,13 @@ func (p *ScrapedPerformer) ToPartial(endpoint string, excluded map[string]bool,
}
if p.CareerLength != nil && !excluded["career_length"] {
// parse career_length into career_start/career_end
start, end, err := utils.ParseYearRangeString(*p.CareerLength)
start, end, err := ParseYearRangeString(*p.CareerLength)
if err == nil {
if start != nil {
ret.CareerStart = NewOptionalInt(*start)
ret.CareerStart = NewOptionalDate(*start)
}
if end != nil {
ret.CareerEnd = NewOptionalInt(*end)
ret.CareerEnd = NewOptionalDate(*end)
}
}
}
@ -471,11 +475,12 @@ func (p *ScrapedPerformer) ToPartial(endpoint string, excluded map[string]bool,
type ScrapedTag struct {
// Set if tag matched
StoredID *string `json:"stored_id"`
Name string `json:"name"`
Description *string `json:"description"`
AliasList []string `json:"alias_list"`
RemoteSiteID *string `json:"remote_site_id"`
StoredID *string `json:"stored_id"`
Name string `json:"name"`
Description *string `json:"description"`
AliasList []string `json:"alias_list"`
RemoteSiteID *string `json:"remote_site_id"`
Parent *ScrapedTag `json:"parent"`
}
func (ScrapedTag) IsScrapedContent() {}
@ -496,6 +501,13 @@ func (t *ScrapedTag) ToTag(endpoint string, excluded map[string]bool) *Tag {
ret.Aliases = NewRelatedStrings(t.AliasList)
}
if t.Parent != nil && t.Parent.StoredID != nil {
parentID, err := strconv.Atoi(*t.Parent.StoredID)
if err == nil && parentID > 0 {
ret.ParentIDs = NewRelatedIDs([]int{parentID})
}
}
if t.RemoteSiteID != nil && endpoint != "" && *t.RemoteSiteID != "" {
ret.StashIDs = NewRelatedStashIDs([]StashID{
{
@ -527,6 +539,16 @@ func (t *ScrapedTag) ToPartial(storedID string, endpoint string, excluded map[st
}
}
if t.Parent != nil && t.Parent.StoredID != nil {
parentID, err := strconv.Atoi(*t.Parent.StoredID)
if err == nil && parentID > 0 {
ret.ParentIDs = &UpdateIDs{
IDs: []int{parentID},
Mode: RelationshipUpdateModeAdd,
}
}
}
if t.RemoteSiteID != nil && endpoint != "" && *t.RemoteSiteID != "" {
ret.StashIDs = &UpdateStashIDs{
StashIDs: existingStashIDs,

10
pkg/models/model_scraped_item_test.go
View file

@ -8,8 +8,6 @@ import (
"github.com/stretchr/testify/assert"
)
func intPtr(i int) *int { return &i }
func Test_scrapedToStudioInput(t *testing.T) {
const name = "name"
url := "url"
@ -186,8 +184,8 @@ func Test_scrapedToPerformerInput(t *testing.T) {
Weight: nextVal(),
Measurements: nextVal(),
FakeTits: nextVal(),
CareerStart: intPtr(2005),
CareerEnd: intPtr(2015),
CareerStart: dateStrFromInt(2005),
CareerEnd: dateStrFromInt(2015),
Tattoos: nextVal(),
Piercings: nextVal(),
Aliases: nextVal(),
@ -212,8 +210,8 @@ func Test_scrapedToPerformerInput(t *testing.T) {
Weight: nextIntVal(),
Measurements: *nextVal(),
FakeTits: *nextVal(),
CareerStart: intPtr(2005),
CareerEnd: intPtr(2015),
CareerStart: dateFromInt(2005),
CareerEnd: dateFromInt(2015),
Tattoos: *nextVal(), // skip CareerLength counter slot
Piercings: *nextVal(),
Aliases: NewRelatedStrings([]string{*nextVal()}),

138
pkg/models/performer.go
View file

@ -61,49 +61,49 @@ type GenderCriterionInput struct {
Modifier CriterionModifier `json:"modifier"`
}
type CircumisedEnum string
type CircumcisedEnum string
const (
CircumisedEnumCut CircumisedEnum = "CUT"
CircumisedEnumUncut CircumisedEnum = "UNCUT"
CircumcisedEnumCut CircumcisedEnum = "CUT"
CircumcisedEnumUncut CircumcisedEnum = "UNCUT"
)
var AllCircumcisionEnum = []CircumisedEnum{
CircumisedEnumCut,
CircumisedEnumUncut,
var AllCircumcisionEnum = []CircumcisedEnum{
CircumcisedEnumCut,
CircumcisedEnumUncut,
}
func (e CircumisedEnum) IsValid() bool {
func (e CircumcisedEnum) IsValid() bool {
switch e {
case CircumisedEnumCut, CircumisedEnumUncut:
case CircumcisedEnumCut, CircumcisedEnumUncut:
return true
}
return false
}
func (e CircumisedEnum) String() string {
func (e CircumcisedEnum) String() string {
return string(e)
}
func (e *CircumisedEnum) UnmarshalGQL(v interface{}) error {
func (e *CircumcisedEnum) UnmarshalGQL(v interface{}) error {
str, ok := v.(string)
if !ok {
return fmt.Errorf("enums must be strings")
}
*e = CircumisedEnum(str)
*e = CircumcisedEnum(str)
if !e.IsValid() {
return fmt.Errorf("%s is not a valid CircumisedEnum", str)
return fmt.Errorf("%s is not a valid CircumcisedEnum", str)
}
return nil
}
func (e CircumisedEnum) MarshalGQL(w io.Writer) {
func (e CircumcisedEnum) MarshalGQL(w io.Writer) {
fmt.Fprint(w, strconv.Quote(e.String()))
}
type CircumcisionCriterionInput struct {
Value []CircumisedEnum `json:"value"`
Value []CircumcisedEnum `json:"value"`
Modifier CriterionModifier `json:"modifier"`
}
@ -139,9 +139,9 @@ type PerformerFilterType struct {
// Filter by career length
CareerLength *StringCriterionInput `json:"career_length"` // deprecated
// Filter by career start year
CareerStart *IntCriterionInput `json:"career_start"`
CareerStart *DateCriterionInput `json:"career_start"`
// Filter by career end year
CareerEnd *IntCriterionInput `json:"career_end"`
CareerEnd *DateCriterionInput `json:"career_end"`
// Filter by tattoos
Tattoos *StringCriterionInput `json:"tattoos"`
// Filter by piercings
@ -216,32 +216,32 @@ type PerformerFilterType struct {
}
type PerformerCreateInput struct {
Name string `json:"name"`
Disambiguation *string `json:"disambiguation"`
URL *string `json:"url"` // deprecated
Urls []string `json:"urls"`
Gender *GenderEnum `json:"gender"`
Birthdate *string `json:"birthdate"`
Ethnicity *string `json:"ethnicity"`
Country *string `json:"country"`
EyeColor *string `json:"eye_color"`
Height *string `json:"height"`
HeightCm *int `json:"height_cm"`
Measurements *string `json:"measurements"`
FakeTits *string `json:"fake_tits"`
PenisLength *float64 `json:"penis_length"`
Circumcised *CircumisedEnum `json:"circumcised"`
CareerLength *string `json:"career_length"`
CareerStart *int `json:"career_start"`
CareerEnd *int `json:"career_end"`
Tattoos *string `json:"tattoos"`
Piercings *string `json:"piercings"`
Aliases *string `json:"aliases"`
AliasList []string `json:"alias_list"`
Twitter *string `json:"twitter"` // deprecated
Instagram *string `json:"instagram"` // deprecated
Favorite *bool `json:"favorite"`
TagIds []string `json:"tag_ids"`
Name string `json:"name"`
Disambiguation *string `json:"disambiguation"`
URL *string `json:"url"` // deprecated
Urls []string `json:"urls"`
Gender *GenderEnum `json:"gender"`
Birthdate *string `json:"birthdate"`
Ethnicity *string `json:"ethnicity"`
Country *string `json:"country"`
EyeColor *string `json:"eye_color"`
Height *string `json:"height"`
HeightCm *int `json:"height_cm"`
Measurements *string `json:"measurements"`
FakeTits *string `json:"fake_tits"`
PenisLength *float64 `json:"penis_length"`
Circumcised *CircumcisedEnum `json:"circumcised"`
CareerLength *string `json:"career_length"`
CareerStart *string `json:"career_start"`
CareerEnd *string `json:"career_end"`
Tattoos *string `json:"tattoos"`
Piercings *string `json:"piercings"`
Aliases *string `json:"aliases"`
AliasList []string `json:"alias_list"`
Twitter *string `json:"twitter"` // deprecated
Instagram *string `json:"instagram"` // deprecated
Favorite *bool `json:"favorite"`
TagIds []string `json:"tag_ids"`
// This should be a URL or a base64 encoded data URL
Image *string `json:"image"`
StashIds []StashIDInput `json:"stash_ids"`
@ -256,33 +256,33 @@ type PerformerCreateInput struct {
}
type PerformerUpdateInput struct {
ID string `json:"id"`
Name *string `json:"name"`
Disambiguation *string `json:"disambiguation"`
URL *string `json:"url"` // deprecated
Urls []string `json:"urls"`
Gender *GenderEnum `json:"gender"`
Birthdate *string `json:"birthdate"`
Ethnicity *string `json:"ethnicity"`
Country *string `json:"country"`
EyeColor *string `json:"eye_color"`
Height *string `json:"height"`
HeightCm *int `json:"height_cm"`
Measurements *string `json:"measurements"`
FakeTits *string `json:"fake_tits"`
PenisLength *float64 `json:"penis_length"`
Circumcised *CircumisedEnum `json:"circumcised"`
CareerLength *string `json:"career_length"`
CareerStart *int `json:"career_start"`
CareerEnd *int `json:"career_end"`
Tattoos *string `json:"tattoos"`
Piercings *string `json:"piercings"`
Aliases *string `json:"aliases"`
AliasList []string `json:"alias_list"`
Twitter *string `json:"twitter"` // deprecated
Instagram *string `json:"instagram"` // deprecated
Favorite *bool `json:"favorite"`
TagIds []string `json:"tag_ids"`
ID string `json:"id"`
Name *string `json:"name"`
Disambiguation *string `json:"disambiguation"`
URL *string `json:"url"` // deprecated
Urls []string `json:"urls"`
Gender *GenderEnum `json:"gender"`
Birthdate *string `json:"birthdate"`
Ethnicity *string `json:"ethnicity"`
Country *string `json:"country"`
EyeColor *string `json:"eye_color"`
Height *string `json:"height"`
HeightCm *int `json:"height_cm"`
Measurements *string `json:"measurements"`
FakeTits *string `json:"fake_tits"`
PenisLength *float64 `json:"penis_length"`
Circumcised *CircumcisedEnum `json:"circumcised"`
CareerLength *string `json:"career_length"`
CareerStart *string `json:"career_start"`
CareerEnd *string `json:"career_end"`
Tattoos *string `json:"tattoos"`
Piercings *string `json:"piercings"`
Aliases *string `json:"aliases"`
AliasList []string `json:"alias_list"`
Twitter *string `json:"twitter"` // deprecated
Instagram *string `json:"instagram"` // deprecated
Favorite *bool `json:"favorite"`
TagIds []string `json:"tag_ids"`
// This should be a URL or a base64 encoded data URL
Image *string `json:"image"`
StashIds []StashIDInput `json:"stash_ids"`

2
pkg/models/repository_file.go
View file

@ -14,7 +14,7 @@ type FileGetter interface {
type FileFinder interface {
FileGetter
FindAllByPath(ctx context.Context, path string, caseSensitive bool) ([]File, error)
FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]File, error)
FindAllInPaths(ctx context.Context, p []string, includeZipContents bool, limit, offset int) ([]File, error)
FindByPath(ctx context.Context, path string, caseSensitive bool) (File, error)
FindByFingerprint(ctx context.Context, fp Fingerprint) ([]File, error)
FindByZipFileID(ctx context.Context, zipFileID FileID) ([]File, error)

3
pkg/models/repository_folder.go
View file

@ -11,11 +11,12 @@ type FolderGetter interface {
// FolderFinder provides methods to find folders.
type FolderFinder interface {
FolderGetter
FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]*Folder, error)
FindAllInPaths(ctx context.Context, p []string, includeZipContents bool, limit, offset int) ([]*Folder, error)
FindByPath(ctx context.Context, path string, caseSensitive bool) (*Folder, error)
FindByZipFileID(ctx context.Context, zipFileID FileID) ([]*Folder, error)
FindByParentFolderID(ctx context.Context, parentFolderID FolderID) ([]*Folder, error)
GetManyParentFolderIDs(ctx context.Context, folderIDs []FolderID) ([][]FolderID, error)
GetManySubFolderIDs(ctx context.Context, folderIDs []FolderID) ([][]FolderID, error)
}
type FolderQueryer interface {

1
pkg/models/repository_gallery.go
View file

@ -83,6 +83,7 @@ type GalleryWriter interface {
CustomFieldsWriter
AddSceneIDs(ctx context.Context, galleryID int, sceneIDs []int) error
AddFileID(ctx context.Context, id int, fileID FileID) error
AddImages(ctx context.Context, galleryID int, imageIDs ...int) error
RemoveImages(ctx context.Context, galleryID int, imageIDs ...int) error

9
pkg/models/repository_tag.go
View file

@ -9,9 +9,16 @@ type TagGetter interface {
Find(ctx context.Context, id int) (*Tag, error)
}
type TagNameFinder interface {
FindByName(ctx context.Context, name string, nocase bool) (*Tag, error)
FindByNames(ctx context.Context, names []string, nocase bool) ([]*Tag, error)
FindByAlias(ctx context.Context, alias string, nocase bool) (*Tag, error)
}
// TagFinder provides methods to find tags.
type TagFinder interface {
TagGetter
TagNameFinder
FindAllAncestors(ctx context.Context, tagID int, excludeIDs []int) ([]*TagPath, error)
FindAllDescendants(ctx context.Context, tagID int, excludeIDs []int) ([]*TagPath, error)
FindByParentTagID(ctx context.Context, parentID int) ([]*Tag, error)
@ -23,8 +30,6 @@ type TagFinder interface {
FindByGroupID(ctx context.Context, groupID int) ([]*Tag, error)
FindBySceneMarkerID(ctx context.Context, sceneMarkerID int) ([]*Tag, error)
FindByStudioID(ctx context.Context, studioID int) ([]*Tag, error)
FindByName(ctx context.Context, name string, nocase bool) (*Tag, error)
FindByNames(ctx context.Context, names []string, nocase bool) ([]*Tag, error)
FindByStashID(ctx context.Context, stashID StashID) ([]*Tag, error)
FindByStashIDStatus(ctx context.Context, hasStashID bool, stashboxEndpoint string) ([]*Tag, error)
}

4
pkg/performer/export.go
View file

@ -71,10 +71,10 @@ func ToJSON(ctx context.Context, reader ImageAliasStashIDGetter, performer *mode
}
if performer.CareerStart != nil {
newPerformerJSON.CareerStart = performer.CareerStart
newPerformerJSON.CareerStart = performer.CareerStart.String()
}
if performer.CareerEnd != nil {
newPerformerJSON.CareerEnd = performer.CareerEnd
newPerformerJSON.CareerEnd = performer.CareerEnd.String()
}
if err := performer.LoadAliases(ctx, reader); err != nil {

10
pkg/performer/export_test.go
View file

@ -48,10 +48,10 @@ var (
rating = 5
height = 123
weight = 60
careerStart = 2005
careerEnd = 2015
careerStart, _ = models.ParseDate("2005")
careerEnd, _ = models.ParseDate("2015")
penisLength = 1.23
circumcisedEnum = models.CircumisedEnumCut
circumcisedEnum = models.CircumcisedEnumCut
circumcised = circumcisedEnum.String()
emptyCustomFields = make(map[string]interface{})
@ -134,8 +134,8 @@ func createFullJSONPerformer(name string, image string, withCustomFields bool) *
URLs: []string{url, twitter, instagram},
Aliases: aliases,
Birthdate: birthDate.String(),
CareerStart: &careerStart,
CareerEnd: &careerEnd,
CareerStart: careerStart.String(),
CareerEnd: careerEnd.String(),
Country: country,
Ethnicity: ethnicity,
EyeColor: eyeColor,

16
pkg/performer/import.go
View file

@ -247,7 +247,7 @@ func performerJSONToPerformer(performerJSON jsonschema.Performer) (models.Perfor
}
if performerJSON.Circumcised != "" {
v := models.CircumisedEnum(performerJSON.Circumcised)
v := models.CircumcisedEnum(performerJSON.Circumcised)
newPerformer.Circumcised = &v
}
@ -285,11 +285,17 @@ func performerJSONToPerformer(performerJSON jsonschema.Performer) (models.Perfor
}
// prefer explicit career_start/career_end, fall back to parsing legacy career_length
if performerJSON.CareerStart != nil || performerJSON.CareerEnd != nil {
newPerformer.CareerStart = performerJSON.CareerStart
newPerformer.CareerEnd = performerJSON.CareerEnd
if performerJSON.CareerStart != "" || performerJSON.CareerEnd != "" {
careerStart, err := models.ParseDate(performerJSON.CareerStart)
if err == nil {
newPerformer.CareerStart = &careerStart
}
careerEnd, err := models.ParseDate(performerJSON.CareerEnd)
if err == nil {
newPerformer.CareerEnd = &careerEnd
}
} else if performerJSON.CareerLength != "" {
start, end, err := utils.ParseYearRangeString(performerJSON.CareerLength)
start, end, err := models.ParseYearRangeString(performerJSON.CareerLength)
if err != nil {
return models.Performer{}, fmt.Errorf("invalid career_length %q: %w", performerJSON.CareerLength, err)
}

12
pkg/performer/import_test.go
View file

@ -317,15 +317,15 @@ func TestUpdate(t *testing.T) {
}
func TestImportCareerFields(t *testing.T) {
startYear := 2005
endYear := 2015
startYear, _ := models.ParseDate("2005")
endYear, _ := models.ParseDate("2015")
// explicit career_start/career_end should be used directly
t.Run("explicit fields", func(t *testing.T) {
input := jsonschema.Performer{
Name: "test",
CareerStart: &startYear,
CareerEnd: &endYear,
CareerStart: startYear.String(),
CareerEnd: endYear.String(),
}
p, err := performerJSONToPerformer(input)
@ -338,8 +338,8 @@ func TestImportCareerFields(t *testing.T) {
t.Run("explicit fields override legacy", func(t *testing.T) {
input := jsonschema.Performer{
Name: "test",
CareerStart: &startYear,
CareerEnd: &endYear,
CareerStart: startYear.String(),
CareerEnd: endYear.String(),
CareerLength: "1990 - 1995",
}

32
pkg/pkg/cache.go
View file

@ -1,6 +1,7 @@
package pkg
import (
"sync"
"time"
)
@ -10,22 +11,23 @@ type cacheEntry struct {
}
type repositoryCache struct {
mu sync.RWMutex
// cache maps the URL to the last modified time and the data
cache map[string]cacheEntry
}
func (c *repositoryCache) ensureCache() {
if c.cache == nil {
c.cache = make(map[string]cacheEntry)
}
}
func (c *repositoryCache) lastModified(url string) *time.Time {
if c == nil {
return nil
}
c.ensureCache()
c.mu.RLock()
defer c.mu.RUnlock()
if c.cache == nil {
return nil
}
e, found := c.cache[url]
if !found {
@ -36,7 +38,13 @@ func (c *repositoryCache) lastModified(url string) *time.Time {
}
func (c *repositoryCache) getPackageList(url string) []RemotePackage {
c.ensureCache()
c.mu.RLock()
defer c.mu.RUnlock()
if c.cache == nil {
return nil
}
e, found := c.cache[url]
if !found {
@ -51,7 +59,13 @@ func (c *repositoryCache) cacheList(url string, lastModified time.Time, data []R
return
}
c.ensureCache()
c.mu.Lock()
defer c.mu.Unlock()
if c.cache == nil {
c.cache = make(map[string]cacheEntry)
}
c.cache[url] = cacheEntry{
lastModified: lastModified,
data: data,

8
pkg/pkg/manager.go
View file

@ -10,6 +10,7 @@ import (
"net/http"
"net/url"
"path/filepath"
"sync"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
@ -31,13 +32,14 @@ type Manager struct {
Client *http.Client
cache *repositoryCache
cacheOnce sync.Once
cache *repositoryCache
}
func (m *Manager) getCache() *repositoryCache {
if m.cache == nil {
m.cacheOnce.Do(func() {
m.cache = &repositoryCache{}
}
})
return m.cache
}

6
pkg/scene/filename_parser.go
View file

@ -456,7 +456,7 @@ type FilenameParserRepository struct {
Performer PerformerNamesFinder
Studio models.StudioQueryer
Group GroupNameFinder
Tag models.TagQueryer
Tag models.TagNameFinder
}
func NewFilenameParserRepository(repo models.Repository) FilenameParserRepository {
@ -599,7 +599,7 @@ func (p *FilenameParser) queryGroup(ctx context.Context, qb GroupNameFinder, gro
return ret
}
func (p *FilenameParser) queryTag(ctx context.Context, qb models.TagQueryer, tagName string) *models.Tag {
func (p *FilenameParser) queryTag(ctx context.Context, qb models.TagNameFinder, tagName string) *models.Tag {
// massage the tag name
tagName = delimiterRE.ReplaceAllString(tagName, " ")
@ -638,7 +638,7 @@ func (p *FilenameParser) setPerformers(ctx context.Context, qb PerformerNamesFin
}
}
func (p *FilenameParser) setTags(ctx context.Context, qb models.TagQueryer, h sceneHolder, result *models.SceneParserResult) {
func (p *FilenameParser) setTags(ctx context.Context, qb models.TagNameFinder, h sceneHolder, result *models.SceneParserResult) {
// query for each performer
tagsSet := make(map[int]bool)
for _, tagName := range h.tags {

40
pkg/scene/scan.go
View file

@ -4,6 +4,8 @@ import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"github.com/stashapp/stash/pkg/file/video"
"github.com/stashapp/stash/pkg/logger"
@ -32,12 +34,18 @@ type ScanCreatorUpdater interface {
AddFileID(ctx context.Context, id int, fileID models.FileID) error
}
type ScanGalleryFinderUpdater interface {
FindByPath(ctx context.Context, p string) ([]*models.Gallery, error)
AddSceneIDs(ctx context.Context, galleryID int, sceneIDs []int) error
}
type ScanGenerator interface {
Generate(ctx context.Context, s *models.Scene, f *models.VideoFile) error
}
type ScanHandler struct {
CreatorUpdater ScanCreatorUpdater
CreatorUpdater ScanCreatorUpdater
GalleryFinderUpdater ScanGalleryFinderUpdater
ScanGenerator ScanGenerator
CaptionUpdater video.CaptionUpdater
@ -127,6 +135,10 @@ func (h *ScanHandler) Handle(ctx context.Context, f models.File, oldFile models.
}
}
if err := h.associateGallery(ctx, existing, f); err != nil {
return err
}
// do this after the commit so that cover generation doesn't hold up the transaction
txn.AddPostCommitHook(ctx, func(ctx context.Context) {
for _, s := range existing {
@ -175,3 +187,29 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
return nil
}
func (h *ScanHandler) associateGallery(ctx context.Context, existing []*models.Scene, f models.File) error {
sceneIDs := make([]int, len(existing))
for i, s := range existing {
sceneIDs[i] = s.ID
}
path := f.Base().Path
zipPath := strings.TrimSuffix(path, filepath.Ext(path)) + ".zip"
// find galleries with a file that matches
galleries, err := h.GalleryFinderUpdater.FindByPath(ctx, zipPath)
if err != nil {
return err
}
for _, gallery := range galleries {
// found related Scene
logger.Infof("associate: Scene %s is related to gallery: %d", path, gallery.ID)
if err := h.GalleryFinderUpdater.AddSceneIDs(ctx, gallery.ID, sceneIDs); err != nil {
return err
}
}
return nil
}

1
pkg/scraper/cache.go
View file

@ -70,6 +70,7 @@ type StudioFinder interface {
type TagFinder interface {
models.TagGetter
models.TagNameFinder
models.TagAutoTagQueryer
}

4
pkg/scraper/mapped_result.go
View file

@ -140,8 +140,8 @@ func (r mappedResult) scrapedPerformer() *models.ScrapedPerformer {
PenisLength: r.stringPtr("PenisLength"),
Circumcised: r.stringPtr("Circumcised"),
CareerLength: r.stringPtr("CareerLength"),
CareerStart: r.IntPtr("CareerStart"),
CareerEnd: r.IntPtr("CareerEnd"),
CareerStart: r.stringPtr("CareerStart"),
CareerEnd: r.stringPtr("CareerEnd"),
Tattoos: r.stringPtr("Tattoos"),
Piercings: r.stringPtr("Piercings"),
Aliases: r.stringPtr("Aliases"),

4
pkg/scraper/performer.go
View file

@ -20,8 +20,8 @@ type ScrapedPerformerInput struct {
PenisLength *string `json:"penis_length"`
Circumcised *string `json:"circumcised"`
CareerLength *string `json:"career_length"`
CareerStart *int `json:"career_start"`
CareerEnd *int `json:"career_end"`
CareerStart *string `json:"career_start"`
CareerEnd *string `json:"career_end"`
Tattoos *string `json:"tattoos"`
Piercings *string `json:"piercings"`
Aliases *string `json:"aliases"`

144
pkg/scraper/post_processing_test.go Normal file
View file

@ -0,0 +1,144 @@
package scraper
import (
"context"
"testing"
"github.com/stashapp/stash/pkg/models"
)
func TestPostScrapePerformerCareerLength(t *testing.T) {
ctx := context.Background()
const related = false
strPtr := func(s string) *string {
return &s
}
tests := []struct {
name string
input models.ScrapedPerformer
want models.ScrapedPerformer
}{
{
"start = 2000",
models.ScrapedPerformer{
CareerStart: strPtr("2000"),
},
models.ScrapedPerformer{
CareerStart: strPtr("2000"),
CareerLength: strPtr("2000 -"),
},
},
{
"end = 2000",
models.ScrapedPerformer{
CareerEnd: strPtr("2000"),
},
models.ScrapedPerformer{
CareerEnd: strPtr("2000"),
CareerLength: strPtr("- 2000"),
},
},
{
"start = 2000, end = 2020",
models.ScrapedPerformer{
CareerStart: strPtr("2000"),
CareerEnd: strPtr("2020"),
},
models.ScrapedPerformer{
CareerStart: strPtr("2000"),
CareerEnd: strPtr("2020"),
CareerLength: strPtr("2000 - 2020"),
},
},
{
"length = 2000 -",
models.ScrapedPerformer{
CareerLength: strPtr("2000 -"),
},
models.ScrapedPerformer{
CareerStart: strPtr("2000"),
CareerLength: strPtr("2000 -"),
},
},
{
"length = - 2010",
models.ScrapedPerformer{
CareerLength: strPtr("- 2010"),
},
models.ScrapedPerformer{
CareerEnd: strPtr("2010"),
CareerLength: strPtr("- 2010"),
},
},
{
"length = 2000 - 2010",
models.ScrapedPerformer{
CareerLength: strPtr("2000 - 2010"),
},
models.ScrapedPerformer{
CareerStart: strPtr("2000"),
CareerEnd: strPtr("2010"),
CareerLength: strPtr("2000 - 2010"),
},
},
{
"invalid start",
models.ScrapedPerformer{
CareerStart: strPtr("two thousand"),
},
models.ScrapedPerformer{
CareerStart: strPtr("two thousand"),
},
},
{
"invalid end",
models.ScrapedPerformer{
CareerEnd: strPtr("two thousand"),
},
models.ScrapedPerformer{
CareerEnd: strPtr("two thousand"),
},
},
{
"invalid career length",
models.ScrapedPerformer{
CareerLength: strPtr("1234 - 4567 - 9224"),
},
models.ScrapedPerformer{
CareerLength: strPtr("1234 - 4567 - 9224"),
},
},
}
compareStrPtr := func(a, b *string) bool {
if a == b {
return true
}
if a == nil || b == nil {
return false
}
return *a == *b
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &postScraper{}
got, err := c.postScrapePerformer(ctx, tt.input, related)
if err != nil {
t.Fatalf("postScrapePerformer returned error: %v", err)
}
postScraped := got.(models.ScrapedPerformer)
if !compareStrPtr(postScraped.CareerStart, tt.want.CareerStart) {
t.Errorf("CareerStart = %v, want %v", postScraped.CareerStart, tt.want.CareerStart)
}
if !compareStrPtr(postScraped.CareerEnd, tt.want.CareerEnd) {
t.Errorf("CareerEnd = %v, want %v", postScraped.CareerEnd, tt.want.CareerEnd)
}
if !compareStrPtr(postScraped.CareerLength, tt.want.CareerLength) {
t.Errorf("CareerLength = %v, want %v", postScraped.CareerLength, tt.want.CareerLength)
}
})
}
}

67
pkg/scraper/postprocessing.go
View file

@ -125,23 +125,64 @@ func (c *postScraper) postScrapePerformer(ctx context.Context, p models.ScrapedP
}
}
isEmptyStr := func(s *string) bool { return s == nil || *s == "" }
isEmptyInt := func(s *int) bool { return s == nil || *s == 0 }
// populate career start/end from career length and vice versa
if !isEmptyStr(p.CareerLength) && isEmptyInt(p.CareerStart) && isEmptyInt(p.CareerEnd) {
p.CareerStart, p.CareerEnd, err = utils.ParseYearRangeString(*p.CareerLength)
if err != nil {
logger.Warnf("Could not parse career length %s: %v", *p.CareerLength, err)
}
} else if isEmptyStr(p.CareerLength) && (!isEmptyInt(p.CareerStart) || !isEmptyInt(p.CareerEnd)) {
v := utils.FormatYearRange(p.CareerStart, p.CareerEnd)
p.CareerLength = &v
}
c.postProcessCareerLength(&p)
return p, nil
}
func (c *postScraper) postProcessCareerLength(p *models.ScrapedPerformer) {
isEmptyStr := func(s *string) bool { return s == nil || *s == "" }
// populate career start/end from career length and vice versa
if !isEmptyStr(p.CareerLength) && isEmptyStr(p.CareerStart) && isEmptyStr(p.CareerEnd) {
start, end, err := models.ParseYearRangeString(*p.CareerLength)
if err != nil {
logger.Warnf("Could not parse career length %s: %v", *p.CareerLength, err)
return
}
if start != nil {
startStr := start.String()
p.CareerStart = &startStr
}
if end != nil {
endStr := end.String()
p.CareerEnd = &endStr
}
return
}
// populate career length from career start/end if career length is missing
if isEmptyStr(p.CareerLength) {
var (
start *models.Date
end *models.Date
)
if !isEmptyStr(p.CareerStart) {
date, err := models.ParseDate(*p.CareerStart)
if err != nil {
logger.Warnf("Could not parse career start %s: %v", *p.CareerStart, err)
return
}
start = &date
}
if !isEmptyStr(p.CareerEnd) {
date, err := models.ParseDate(*p.CareerEnd)
if err != nil {
logger.Warnf("Could not parse career end %s: %v", *p.CareerEnd, err)
return
}
end = &date
}
v := models.FormatYearRange(start, end)
p.CareerLength = &v
}
}
func (c *postScraper) postScrapeMovie(ctx context.Context, m models.ScrapedMovie, related bool) (_ ScrapedContent, err error) {
r := c.repository
tqb := r.TagFinder

6
pkg/scraper/query_url.go
View file

@ -17,6 +17,12 @@ func queryURLParametersFromScene(scene *models.Scene) queryURLParameters {
ret["oshash"] = scene.OSHash
ret["filename"] = filepath.Base(scene.Path)
// pull phash from primary file
phashFingerprints := scene.Files.Primary().Base().Fingerprints.Filter(models.FingerprintTypePhash)
if len(phashFingerprints) > 0 {
ret["phash"] = phashFingerprints[0].Value()
}
if scene.Title != "" {
ret["title"] = scene.Title
}

2
pkg/scraper/tag.go
View file

@ -11,7 +11,7 @@ import (
"github.com/stashapp/stash/pkg/sliceutil"
)
func postProcessTags(ctx context.Context, tqb models.TagQueryer, scrapedTags []*models.ScrapedTag) (ret []*models.ScrapedTag, err error) {
func postProcessTags(ctx context.Context, tqb models.TagNameFinder, scrapedTags []*models.ScrapedTag) (ret []*models.ScrapedTag, err error) {
ret = make([]*models.ScrapedTag, 0, len(scrapedTags))
for _, t := range scrapedTags {

10
pkg/sqlite/custom_fields.go
View file

@ -261,8 +261,8 @@ func (h *customFieldsFilterHandler) handleCriterion(f *filterBuilder, joinAs str
h.innerJoin(f, joinAs, cc.Field)
f.addWhere(fmt.Sprintf("%[1]s.value IN %s", joinAs, getInBinding(len(cv))), cv...)
case models.CriterionModifierNotEquals:
h.innerJoin(f, joinAs, cc.Field)
f.addWhere(fmt.Sprintf("%[1]s.value NOT IN %s", joinAs, getInBinding(len(cv))), cv...)
h.leftJoin(f, joinAs, cc.Field)
f.addWhere(fmt.Sprintf("(%[1]s.value NOT IN %s OR %[1]s.value IS NULL)", joinAs, getInBinding(len(cv))), cv...)
case models.CriterionModifierIncludes:
clauses := make([]sqlClause, len(cv))
for i, v := range cv {
@ -272,7 +272,7 @@ func (h *customFieldsFilterHandler) handleCriterion(f *filterBuilder, joinAs str
f.whereClauses = append(f.whereClauses, clauses...)
case models.CriterionModifierExcludes:
for _, v := range cv {
f.addWhere(fmt.Sprintf("%[1]s.value NOT LIKE ?", joinAs), fmt.Sprintf("%%%v%%", v))
f.addWhere(fmt.Sprintf("(%[1]s.value NOT LIKE ? OR %[1]s.value IS NULL)", joinAs), fmt.Sprintf("%%%v%%", v))
}
h.leftJoin(f, joinAs, cc.Field)
case models.CriterionModifierMatchesRegex:
@ -315,8 +315,8 @@ func (h *customFieldsFilterHandler) handleCriterion(f *filterBuilder, joinAs str
h.innerJoin(f, joinAs, cc.Field)
f.addWhere(fmt.Sprintf("%s.value BETWEEN ? AND ?", joinAs), cv[0], cv[1])
case models.CriterionModifierNotBetween:
h.innerJoin(f, joinAs, cc.Field)
f.addWhere(fmt.Sprintf("%s.value NOT BETWEEN ? AND ?", joinAs), cv[0], cv[1])
h.leftJoin(f, joinAs, cc.Field)
f.addWhere(fmt.Sprintf("(%s.value NOT BETWEEN ? AND ? OR %[1]s.value IS NULL)", joinAs), cv[0], cv[1])
case models.CriterionModifierLessThan:
if len(cv) != 1 {
f.setError(fmt.Errorf("expected 1 value for custom field criterion modifier LESS_THAN, got %d", len(cv)))

2
pkg/sqlite/database.go
View file

@ -34,7 +34,7 @@ const (
cacheSizeEnv = "STASH_SQLITE_CACHE_SIZE"
)
var appSchemaVersion uint = 84
var appSchemaVersion uint = 85
//go:embed migrations/*.sql
var migrationsBox embed.FS

6
pkg/sqlite/file.go
View file

@ -695,7 +695,7 @@ func (qb *FileStore) allInPaths(q *goqu.SelectDataset, p []string) *goqu.SelectD
// FindAllByPaths returns the all files that are within any of the given paths.
// Returns all if limit is < 0.
// Returns all files if p is empty.
func (qb *FileStore) FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]models.File, error) {
func (qb *FileStore) FindAllInPaths(ctx context.Context, p []string, includeZipContents bool, limit, offset int) ([]models.File, error) {
table := qb.table()
folderTable := folderTableMgr.table
@ -706,6 +706,10 @@ func (qb *FileStore) FindAllInPaths(ctx context.Context, p []string, limit, offs
q = qb.allInPaths(q, p)
if !includeZipContents {
q = q.Where(table.Col("zip_file_id").IsNull())
}
if limit > -1 {
q = q.Limit(uint(limit))
}

30
pkg/sqlite/file_filter.go
View file

@ -238,22 +238,32 @@ func (qb *fileFilterHandler) hashesCriterionHandler(hashes []*models.Fingerprint
t := fmt.Sprintf("file_fingerprints_%d", i)
f.addLeftJoin(fingerprintTable, t, fmt.Sprintf("files.id = %s.file_id AND %s.type = ?", t, t), hash.Type)
value, _ := utils.StringToPhash(hash.Value)
distance := 0
if hash.Distance != nil {
distance = *hash.Distance
}
if distance > 0 {
// needed to avoid a type mismatch
f.addWhere(fmt.Sprintf("typeof(%s.fingerprint) = 'integer'", t))
f.addWhere(fmt.Sprintf("phash_distance(%s.fingerprint, ?) < ?", t), value, distance)
// Only phash supports distance matching and is stored as integer
if hash.Type == models.FingerprintTypePhash {
value, err := utils.StringToPhash(hash.Value)
if err != nil {
f.setError(fmt.Errorf("invalid phash value: %w", err))
return
}
if distance > 0 {
// needed to avoid a type mismatch
f.addWhere(fmt.Sprintf("typeof(%s.fingerprint) = 'integer'", t))
f.addWhere(fmt.Sprintf("phash_distance(%s.fingerprint, ?) < ?", t), value, distance)
} else {
intCriterionHandler(&models.IntCriterionInput{
Value: int(value),
Modifier: models.CriterionModifierEquals,
}, t+".fingerprint", nil)(ctx, f)
}
} else {
// use the default handler
intCriterionHandler(&models.IntCriterionInput{
Value: int(value),
Modifier: models.CriterionModifierEquals,
}, t+".fingerprint", nil)(ctx, f)
// All other fingerprint types (md5, oshash, sha1, etc.) are stored as strings
// Use exact match for string-based fingerprints
f.addWhere(fmt.Sprintf("%s.fingerprint = ?", t), hash.Value)
}
}
}

41
pkg/sqlite/file_filter_test.go
View file

@ -9,6 +9,7 @@ import (
"testing"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/utils"
"github.com/stretchr/testify/assert"
)
@ -81,7 +82,45 @@ func TestFileQuery(t *testing.T) {
includeIDs: []models.FileID{fileIDs[fileIdxInZip]},
excludeIdxs: []int{fileIdxStartImageFiles},
},
// TODO - add more tests for other file filters
{
name: "hashes md5",
filter: &models.FileFilterType{
Hashes: []*models.FingerprintFilterInput{
{
Type: models.FingerprintTypeMD5,
Value: getPrefixedStringValue("file", fileIdxStartVideoFiles, "md5"),
},
},
},
includeIdxs: []int{fileIdxStartVideoFiles},
excludeIdxs: []int{fileIdxStartImageFiles},
},
{
name: "hashes oshash",
filter: &models.FileFilterType{
Hashes: []*models.FingerprintFilterInput{
{
Type: models.FingerprintTypeOshash,
Value: getPrefixedStringValue("file", fileIdxStartVideoFiles, "oshash"),
},
},
},
includeIdxs: []int{fileIdxStartVideoFiles},
excludeIdxs: []int{fileIdxStartImageFiles},
},
{
name: "hashes phash",
filter: &models.FileFilterType{
Hashes: []*models.FingerprintFilterInput{
{
Type: models.FingerprintTypePhash,
Value: utils.PhashToString(getFilePhash(fileIdxStartImageFiles)),
},
},
},
includeIdxs: []int{fileIdxStartImageFiles},
excludeIdxs: []int{fileIdxStartVideoFiles},
},
}
for _, tt := range tests {

6
pkg/sqlite/file_test.go
View file

@ -572,7 +572,7 @@ func TestFileStore_FindByFingerprint(t *testing.T) {
{
"by MD5",
models.Fingerprint{
Type: "MD5",
Type: models.FingerprintTypeMD5,
Fingerprint: getPrefixedStringValue("file", fileIdxZip, "md5"),
},
[]models.File{makeFileWithID(fileIdxZip)},
@ -581,7 +581,7 @@ func TestFileStore_FindByFingerprint(t *testing.T) {
{
"by OSHASH",
models.Fingerprint{
Type: "OSHASH",
Type: models.FingerprintTypeOshash,
Fingerprint: getPrefixedStringValue("file", fileIdxZip, "oshash"),
},
[]models.File{makeFileWithID(fileIdxZip)},
@ -590,7 +590,7 @@ func TestFileStore_FindByFingerprint(t *testing.T) {
{
"non-existing",
models.Fingerprint{
Type: "OSHASH",
Type: models.FingerprintTypeOshash,
Fingerprint: "foo",
},
nil,

42
pkg/sqlite/folder.go
View file

@ -409,6 +409,42 @@ func (qb *FolderStore) GetManyParentFolderIDs(ctx context.Context, folderIDs []m
return ret, nil
}
func (qb *FolderStore) GetManySubFolderIDs(ctx context.Context, parentFolderIDs []models.FolderID) ([][]models.FolderID, error) {
table := qb.table()
q := dialect.From(table).Select(
table.Col(idColumn),
table.Col("parent_folder_id"),
).Where(qb.table().Col("parent_folder_id").In(parentFolderIDs))
sql, args, err := q.ToSQL()
if err != nil {
return nil, fmt.Errorf("building query: %w", err)
}
var results []struct {
FolderID int `db:"id"`
ParentFolderID models.FolderID `db:"parent_folder_id"`
}
if err := querySelect(ctx, sql, args, &results); err != nil {
return nil, fmt.Errorf("getting folders by parent folder ids %v: %w", parentFolderIDs, err)
}
retMap := make(map[models.FolderID][]models.FolderID)
for _, v := range results {
retMap[v.ParentFolderID] = append(retMap[v.ParentFolderID], models.FolderID(v.FolderID))
}
ret := make([][]models.FolderID, len(parentFolderIDs))
for i, parentID := range parentFolderIDs {
ret[i] = retMap[parentID]
}
return ret, nil
}
func (qb *FolderStore) allInPaths(q *goqu.SelectDataset, p []string) *goqu.SelectDataset {
table := qb.table()
@ -427,10 +463,14 @@ func (qb *FolderStore) allInPaths(q *goqu.SelectDataset, p []string) *goqu.Selec
// FindAllInPaths returns the all folders that are or are within any of the given paths.
// Returns all if limit is < 0.
// Returns all folders if p is empty.
func (qb *FolderStore) FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]*models.Folder, error) {
func (qb *FolderStore) FindAllInPaths(ctx context.Context, p []string, includeZipContents bool, limit, offset int) ([]*models.Folder, error) {
q := qb.selectDataset().Prepared(true)
q = qb.allInPaths(q, p)
if !includeZipContents {
q = q.Where(qb.table().Col("zip_file_id").IsNull())
}
if limit > -1 {
q = q.Limit(uint(limit))
}

4
pkg/sqlite/gallery.go
View file

@ -926,3 +926,7 @@ func (qb *GalleryStore) ResetCover(ctx context.Context, galleryID int) error {
func (qb *GalleryStore) GetSceneIDs(ctx context.Context, id int) ([]int, error) {
return galleryRepository.scenes.getIDs(ctx, id)
}
func (qb *GalleryStore) AddSceneIDs(ctx context.Context, galleryID int, sceneIDs []int) error {
return galleriesScenesTableMgr.insertJoins(ctx, galleryID, sceneIDs)
}

6
pkg/sqlite/gallery_filter.go
View file

@ -285,7 +285,7 @@ func (qb *galleryFilterHandler) parentFolderCriterionHandler(folder *models.Hier
return
}
galleryRepository.addFoldersTable(f)
galleryRepository.addFilesTable(f)
f.addLeftJoin(folderTable, "gallery_folder", "galleries.folder_id = gallery_folder.id")
criterion := *folder
@ -320,7 +320,7 @@ func (qb *galleryFilterHandler) parentFolderCriterionHandler(folder *models.Hier
}
// combine clauses with OR to handle zip file or folder
c1 := makeClause(fmt.Sprintf("folders.parent_folder_id IN (SELECT column2 FROM (%s))", valuesClause))
c1 := makeClause(fmt.Sprintf("files.parent_folder_id IN (SELECT column2 FROM (%s))", valuesClause))
c2 := makeClause(fmt.Sprintf("gallery_folder.parent_folder_id IN (SELECT column2 FROM (%s))", valuesClause))
f.whereClauses = append(f.whereClauses, orClauses(c1, c2))
}
@ -332,7 +332,7 @@ func (qb *galleryFilterHandler) parentFolderCriterionHandler(folder *models.Hier
return
}
f.addWhere(fmt.Sprintf("folders.parent_folder_id NOT IN (SELECT column2 FROM (%s)) OR folders.parent_folder_id IS NULL", valuesClause))
f.addWhere(fmt.Sprintf("files.parent_folder_id NOT IN (SELECT column2 FROM (%s)) OR folders.parent_folder_id IS NULL", valuesClause))
f.addWhere(fmt.Sprintf("gallery_folder.parent_folder_id NOT IN (SELECT column2 FROM (%s)) OR gallery_folder.parent_folder_id IS NULL", valuesClause))
}
}

2
pkg/sqlite/image.go
View file

@ -837,7 +837,7 @@ func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFi
)
filepathColumn := "folders.path || '" + string(filepath.Separator) + "' || files.basename"
searchColumns := []string{"images.title", filepathColumn, "files_fingerprints.fingerprint"}
searchColumns := []string{"images.title", "images.details", filepathColumn, "files_fingerprints.fingerprint"}
query.parseQueryString(searchColumns, *q)
}

14
pkg/sqlite/image_test.go
View file

@ -1596,6 +1596,20 @@ func TestImageQueryQ(t *testing.T) {
})
}
func TestImageQueryQ_Details(t *testing.T) {
withTxn(func(ctx context.Context) error {
const imageIdx = 3
q := getImageStringValue(imageIdx, detailsField)
sqb := db.Image
imageQueryQ(ctx, t, sqb, q, imageIdx)
return nil
})
}
func queryImagesWithCount(ctx context.Context, sqb models.ImageReader, imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) ([]*models.Image, int, error) {
result, err := sqb.Query(ctx, models.ImageQueryOptions{
QueryOptions: models.QueryOptions{

21
pkg/sqlite/migrations/78_postmigrate.go
View file

@ -7,8 +7,8 @@ import (
"github.com/jmoiron/sqlx"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sqlite"
"github.com/stashapp/stash/pkg/utils"
)
type schema78Migrator struct {
@ -76,7 +76,7 @@ func (m *schema78Migrator) migrateCareerLength(ctx context.Context) error {
lastID = id
gotSome = true
start, end, err := utils.ParseYearRangeString(careerLength)
start, end, err := models.ParseYearRangeString(careerLength)
if err != nil {
logger.Warnf("Could not parse career_length %q for performer %d: %v — preserving as custom field", careerLength, id, err)
@ -107,10 +107,23 @@ func (m *schema78Migrator) migrateCareerLength(ctx context.Context) error {
return nil
}
func (m *schema78Migrator) updateCareerFields(tx *sqlx.Tx, id int, start *int, end *int) error {
func (m *schema78Migrator) updateCareerFields(tx *sqlx.Tx, id int, start *models.Date, end *models.Date) error {
var (
startYear, endYear *int
)
if start != nil {
year := start.Year()
startYear = &year
}
if end != nil {
year := end.Year()
endYear = &year
}
_, err := tx.Exec(
"UPDATE performers SET career_start = ?, career_end = ? WHERE id = ?",
start, end, id,
startYear, endYear, id,
)
return err
}

166
pkg/sqlite/migrations/84_postmigrate.go → pkg/sqlite/migrations/84_migrate.go
View file

@ -16,8 +16,8 @@ import (
"gopkg.in/guregu/null.v4"
)
func post84(ctx context.Context, db *sqlx.DB) error {
logger.Info("Running post-migration for schema version 84")
func pre84(ctx context.Context, db *sqlx.DB) error {
logger.Info("Running pre-migration for schema version 84")
m := schema84Migrator{
migrator: migrator{
@ -36,6 +36,23 @@ func post84(ctx context.Context, db *sqlx.DB) error {
return fmt.Errorf("fixing incorrect parent folders: %w", err)
}
if err := m.deduplicateFolders(ctx); err != nil {
return fmt.Errorf("deduplicating folders: %w", err)
}
return nil
}
func post84(ctx context.Context, db *sqlx.DB) error {
logger.Info("Running post-migration for schema version 84")
m := schema84Migrator{
migrator: migrator{
db: db,
},
folderCache: make(map[string]folderInfo),
}
if err := m.migrateFolders(ctx); err != nil {
return fmt.Errorf("migrating folders: %w", err)
}
@ -188,7 +205,7 @@ func (m *schema84Migrator) getOrCreateFolderHierarchy(tx *sqlx.Tx, path string,
logger.Debugf("%s doesn't exist. Creating new folder entry...", path)
// we need to set basename to path, which will be addressed in the next step
const insertSQL = "INSERT INTO `folders` (`path`,`basename`,`parent_folder_id`,`mod_time`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?)"
const insertSQL = "INSERT INTO `folders` (`path`,`parent_folder_id`,`mod_time`,`created_at`,`updated_at`) VALUES (?,?,?,?,?)"
var parentFolderID null.Int
if parentID != nil {
@ -196,7 +213,7 @@ func (m *schema84Migrator) getOrCreateFolderHierarchy(tx *sqlx.Tx, path string,
}
now := time.Now()
result, err := tx.Exec(insertSQL, path, path, parentFolderID, time.Time{}, now, now)
result, err := tx.Exec(insertSQL, path, parentFolderID, time.Time{}, now, now)
if err != nil {
return nil, fmt.Errorf("creating folder %s: %w", path, err)
}
@ -264,11 +281,6 @@ func (m *schema84Migrator) fixIncorrectParents(ctx context.Context, rootPaths []
continue
}
if !logged {
logger.Info("Fixing folders with incorrect parent folder assignments...")
logged = true
}
correctParentID, err := m.getOrCreateFolderHierarchy(tx, expectedParent, rootPaths)
if err != nil {
return fmt.Errorf("error getting/creating correct parent for folder %d %q: %w", id, p, err)
@ -278,6 +290,11 @@ func (m *schema84Migrator) fixIncorrectParents(ctx context.Context, rootPaths []
continue
}
if !logged {
logger.Info("Fixing folders with incorrect parent folder assignments...")
logged = true
}
logger.Debugf("Fixing folder %d %q: changing parent_folder_id from %d to %d", id, p, parentFolderID, *correctParentID)
_, err = tx.Exec("UPDATE `folders` SET `parent_folder_id` = ? WHERE `id` = ?", *correctParentID, id)
@ -309,6 +326,136 @@ func (m *schema84Migrator) fixIncorrectParents(ctx context.Context, rootPaths []
return nil
}
// deduplicateFolders finds folders that would have the same (parent_folder_id, basename) after
// migrateFolders sets basename = filepath.Base(path), and merges the duplicates.
// This can happen when the database contains entries for the same physical folder with different
// path representations (e.g., mixed separators like "\data/movies" vs "\data\movies" on Windows).
func (m *schema84Migrator) deduplicateFolders(ctx context.Context) error {
for {
n, err := m.deduplicateFoldersPass(ctx)
if err != nil {
return err
}
// repeat until no more duplicates are found, since merging child folders
// from a duplicate parent into the canonical parent may create new conflicts
if n == 0 {
break
}
}
return nil
}
func (m *schema84Migrator) deduplicateFoldersPass(ctx context.Context) (int, error) {
type folderRow struct {
ID int `db:"id"`
Path string `db:"path"`
ParentFolderID int `db:"parent_folder_id"`
}
var folders []folderRow
if err := m.db.SelectContext(ctx, &folders,
"SELECT id, path, parent_folder_id FROM folders WHERE parent_folder_id IS NOT NULL ORDER BY id"); err != nil {
return 0, fmt.Errorf("loading folders: %w", err)
}
// group by (parent_folder_id, computed basename)
type groupKey struct {
parentID int
basename string
}
groups := make(map[groupKey][]folderRow)
for _, f := range folders {
key := groupKey{
parentID: f.ParentFolderID,
basename: filepath.Base(f.Path),
}
groups[key] = append(groups[key], f)
}
deduped := 0
for _, group := range groups {
if len(group) <= 1 {
continue
}
if deduped == 0 {
logger.Info("Deduplicating folders with conflicting basenames...")
}
// prefer the folder whose path is already normalized for the current OS,
// falling back to the newest entry (highest ID) since it's most likely
// from the current filesystem
keep := group[len(group)-1]
for _, f := range group {
if f.Path == filepath.Clean(f.Path) {
keep = f
break
}
}
for _, dup := range group {
if dup.ID == keep.ID {
continue
}
logger.Infof("Merging duplicate folder %d %q into folder %d %q", dup.ID, dup.Path, keep.ID, keep.Path)
if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
return m.mergeFolder(tx, keep.ID, dup.ID)
}); err != nil {
return 0, fmt.Errorf("merging folder %d into %d: %w", dup.ID, keep.ID, err)
}
deduped++
}
}
if deduped > 0 {
logger.Infof("Deduplicated %d folder entries", deduped)
}
return deduped, nil
}
func (m *schema84Migrator) mergeFolder(tx *sqlx.Tx, keepID, dupID int) error {
// Re-parent child folders from the duplicate to the canonical folder.
// At this point basenames are still full paths (unique), so this won't cause
// UNIQUE constraint violations on (parent_folder_id, basename).
if _, err := tx.Exec("UPDATE folders SET parent_folder_id = ? WHERE parent_folder_id = ?", keepID, dupID); err != nil {
return fmt.Errorf("re-parenting child folders: %w", err)
}
// re-parent any files under the duplicate folder to the canonical folder.
if _, err := tx.Exec("UPDATE files SET parent_folder_id = ? WHERE parent_folder_id = ?", keepID, dupID); err != nil {
return fmt.Errorf("re-parenting files: %w", err)
}
// delete the duplicate folder entry only if it is not referenced by any galleries
var count int
if err := tx.Get(&count, "SELECT COUNT(*) FROM galleries WHERE folder_id = ?", dupID); err != nil {
return fmt.Errorf("checking for gallery references: %w", err)
}
if count > 0 {
logger.Warnf("Duplicate folder %d is still referenced by %d galleries. Orphaning instead of deleting.", dupID, count)
// Orphan the stale duplicate folder by clearing its parent so the UNIQUE
// constraint on (parent_folder_id, basename) won't be violated when
// migrateFolders sets basenames. Any stale file entries under it are left
// untouched — the clean task will handle them on the next scan.
if _, err := tx.Exec("UPDATE folders SET parent_folder_id = NULL WHERE id = ?", dupID); err != nil {
return fmt.Errorf("orphaning duplicate folder: %w", err)
}
} else {
// delete the duplicate folder entry
if _, err := tx.Exec("DELETE FROM folders WHERE id = ?", dupID); err != nil {
return fmt.Errorf("deleting duplicate folder: %w", err)
}
}
return nil
}
func (m *schema84Migrator) migrateFolders(ctx context.Context) error {
const (
limit = 1000
@ -381,5 +528,6 @@ func (m *schema84Migrator) migrateFolders(ctx context.Context) error {
}
func init() {
sqlite.RegisterPreMigration(84, pre84)
sqlite.RegisterPostMigration(84, post84)
}

112
pkg/sqlite/migrations/85_performer_career_dates.up.sql Normal file
View file

@ -0,0 +1,112 @@
-- have to change the type of the career start/end columns so need to recreate the table
PRAGMA foreign_keys=OFF;
CREATE TABLE IF NOT EXISTS "performers_new" (
`id` integer not null primary key autoincrement,
`name` varchar(255) not null,
`disambiguation` varchar(255),
`gender` varchar(20),
`birthdate` date,
`birthdate_precision` TINYINT,
`ethnicity` varchar(255),
`country` varchar(255),
`eye_color` varchar(255),
`height` int,
`measurements` varchar(255),
`fake_tits` varchar(255),
`tattoos` varchar(255),
`piercings` varchar(255),
`favorite` boolean not null default '0',
`created_at` datetime not null,
`updated_at` datetime not null,
`details` text,
`death_date` date,
`death_date_precision` TINYINT,
`hair_color` varchar(255),
`weight` integer,
`rating` tinyint,
`ignore_auto_tag` boolean not null default '0',
`penis_length` float,
`circumcised` varchar[10],
`career_start` date,
`career_start_precision` TINYINT,
`career_end` date,
`career_end_precision` TINYINT,
`image_blob` varchar(255) REFERENCES `blobs`(`checksum`)
);
INSERT INTO `performers_new` (
`id`,
`name`,
`disambiguation`,
`gender`,
`birthdate`,
`ethnicity`,
`country`,
`eye_color`,
`height`,
`measurements`,
`fake_tits`,
`tattoos`,
`piercings`,
`favorite`,
`created_at`,
`updated_at`,
`details`,
`death_date`,
`hair_color`,
`weight`,
`rating`,
`ignore_auto_tag`,
`image_blob`,
`penis_length`,
`circumcised`,
`birthdate_precision`,
`death_date_precision`,
`career_start`,
`career_end`
) SELECT
`id`,
`name`,
`disambiguation`,
`gender`,
`birthdate`,
`ethnicity`,
`country`,
`eye_color`,
`height`,
`measurements`,
`fake_tits`,
`tattoos`,
`piercings`,
`favorite`,
`created_at`,
`updated_at`,
`details`,
`death_date`,
`hair_color`,
`weight`,
`rating`,
`ignore_auto_tag`,
`image_blob`,
`penis_length`,
`circumcised`,
`birthdate_precision`,
`death_date_precision`,
CAST(`career_start` AS TEXT),
CAST(`career_end` AS TEXT)
FROM `performers`;
DROP INDEX IF EXISTS `performers_name_disambiguation_unique`;
DROP INDEX IF EXISTS `performers_name_unique`;
DROP TABLE `performers`;
ALTER TABLE `performers_new` RENAME TO `performers`;
UPDATE "performers" SET `career_start` = CONCAT(`career_start`, '-01-01'), "career_start_precision" = 2 WHERE "career_start" IS NOT NULL;
UPDATE "performers" SET `career_end` = CONCAT(`career_end`, '-01-01'), "career_end_precision" = 2 WHERE "career_end" IS NOT NULL;
CREATE UNIQUE INDEX `performers_name_disambiguation_unique` on `performers` (`name`, `disambiguation`) WHERE `disambiguation` IS NOT NULL;
CREATE UNIQUE INDEX `performers_name_unique` on `performers` (`name`) WHERE `disambiguation` IS NULL;
PRAGMA foreign_keys=ON;

85
pkg/sqlite/performer.go
View file

@ -30,27 +30,29 @@ const (
)
type performerRow struct {
ID int `db:"id" goqu:"skipinsert"`
Name null.String `db:"name"` // TODO: make schema non-nullable
Disambigation zero.String `db:"disambiguation"`
Gender zero.String `db:"gender"`
Birthdate NullDate `db:"birthdate"`
BirthdatePrecision null.Int `db:"birthdate_precision"`
Ethnicity zero.String `db:"ethnicity"`
Country zero.String `db:"country"`
EyeColor zero.String `db:"eye_color"`
Height null.Int `db:"height"`
Measurements zero.String `db:"measurements"`
FakeTits zero.String `db:"fake_tits"`
PenisLength null.Float `db:"penis_length"`
Circumcised zero.String `db:"circumcised"`
CareerStart null.Int `db:"career_start"`
CareerEnd null.Int `db:"career_end"`
Tattoos zero.String `db:"tattoos"`
Piercings zero.String `db:"piercings"`
Favorite bool `db:"favorite"`
CreatedAt Timestamp `db:"created_at"`
UpdatedAt Timestamp `db:"updated_at"`
ID int `db:"id" goqu:"skipinsert"`
Name null.String `db:"name"` // TODO: make schema non-nullable
Disambigation zero.String `db:"disambiguation"`
Gender zero.String `db:"gender"`
Birthdate NullDate `db:"birthdate"`
BirthdatePrecision null.Int `db:"birthdate_precision"`
Ethnicity zero.String `db:"ethnicity"`
Country zero.String `db:"country"`
EyeColor zero.String `db:"eye_color"`
Height null.Int `db:"height"`
Measurements zero.String `db:"measurements"`
FakeTits zero.String `db:"fake_tits"`
PenisLength null.Float `db:"penis_length"`
Circumcised zero.String `db:"circumcised"`
CareerStart NullDate `db:"career_start"`
CareerStartPrecision null.Int `db:"career_start_precision"`
CareerEnd NullDate `db:"career_end"`
CareerEndPrecision null.Int `db:"career_end_precision"`
Tattoos zero.String `db:"tattoos"`
Piercings zero.String `db:"piercings"`
Favorite bool `db:"favorite"`
CreatedAt Timestamp `db:"created_at"`
UpdatedAt Timestamp `db:"updated_at"`
// expressed as 1-100
Rating null.Int `db:"rating"`
Details zero.String `db:"details"`
@ -83,8 +85,10 @@ func (r *performerRow) fromPerformer(o models.Performer) {
if o.Circumcised != nil && o.Circumcised.IsValid() {
r.Circumcised = zero.StringFrom(o.Circumcised.String())
}
r.CareerStart = intFromPtr(o.CareerStart)
r.CareerEnd = intFromPtr(o.CareerEnd)
r.CareerStart = NullDateFromDatePtr(o.CareerStart)
r.CareerStartPrecision = datePrecisionFromDatePtr(o.CareerStart)
r.CareerEnd = NullDateFromDatePtr(o.CareerEnd)
r.CareerEndPrecision = datePrecisionFromDatePtr(o.CareerEnd)
r.Tattoos = zero.StringFrom(o.Tattoos)
r.Piercings = zero.StringFrom(o.Piercings)
r.Favorite = o.Favorite
@ -112,8 +116,8 @@ func (r *performerRow) resolve() *models.Performer {
Measurements: r.Measurements.String,
FakeTits: r.FakeTits.String,
PenisLength: nullFloatPtr(r.PenisLength),
CareerStart: nullIntPtr(r.CareerStart),
CareerEnd: nullIntPtr(r.CareerEnd),
CareerStart: r.CareerStart.DatePtr(r.CareerStartPrecision),
CareerEnd: r.CareerEnd.DatePtr(r.CareerEndPrecision),
Tattoos: r.Tattoos.String,
Piercings: r.Piercings.String,
Favorite: r.Favorite,
@ -134,7 +138,7 @@ func (r *performerRow) resolve() *models.Performer {
}
if r.Circumcised.ValueOrZero() != "" {
v := models.CircumisedEnum(r.Circumcised.String)
v := models.CircumcisedEnum(r.Circumcised.String)
ret.Circumcised = &v
}
@ -158,8 +162,8 @@ func (r *performerRowRecord) fromPartial(o models.PerformerPartial) {
r.setNullString("fake_tits", o.FakeTits)
r.setNullFloat64("penis_length", o.PenisLength)
r.setNullString("circumcised", o.Circumcised)
r.setNullInt("career_start", o.CareerStart)
r.setNullInt("career_end", o.CareerEnd)
r.setNullDate("career_start", "career_start_precision", o.CareerStart)
r.setNullDate("career_end", "career_end_precision", o.CareerEnd)
r.setNullString("tattoos", o.Tattoos)
r.setNullString("piercings", o.Piercings)
r.setBool("favorite", o.Favorite)
@ -778,6 +782,28 @@ func (qb *PerformerStore) sortByScenesDuration(direction string) string {
return " ORDER BY (" + selectPerformerScenesDurationSQL + ") " + direction
}
// used for sorting by total scene file size
var selectPerformerScenesSizeSQL = utils.StrFormat(
"SELECT COALESCE(SUM({files}.size), 0) FROM {performers_scenes} s "+
"LEFT JOIN {scenes} ON {scenes}.id = s.{scene_id} "+
"LEFT JOIN {scenes_files} ON {scenes_files}.{scene_id} = {scenes}.id "+
"LEFT JOIN {files} ON {files}.id = {scenes_files}.file_id "+
"WHERE s.{performer_id} = {performers}.id",
map[string]interface{}{
"performer_id": performerIDColumn,
"performers": performerTable,
"performers_scenes": performersScenesTable,
"scenes": sceneTable,
"scene_id": sceneIDColumn,
"scenes_files": scenesFilesTable,
"files": fileTable,
},
)
func (qb *PerformerStore) sortByScenesSize(direction string) string {
return " ORDER BY (" + selectPerformerScenesSizeSQL + ") " + direction
}
var performerSortOptions = sortOptions{
"birthdate",
"career_start",
@ -799,6 +825,7 @@ var performerSortOptions = sortOptions{
"rating",
"scenes_count",
"scenes_duration",
"scenes_size",
"tag_count",
"updated_at",
"weight",
@ -828,6 +855,8 @@ func (qb *PerformerStore) getPerformerSort(findFilter *models.FindFilterType) (s
sortQuery += getCountSort(performerTable, performersScenesTable, performerIDColumn, direction)
case "scenes_duration":
sortQuery += qb.sortByScenesDuration(direction)
case "scenes_size":
sortQuery += qb.sortByScenesSize(direction)
case "images_count":
sortQuery += getCountSort(performerTable, performersImagesTable, performerIDColumn, direction)
case "galleries_count":

54
pkg/sqlite/performer_filter.go
View file

@ -52,7 +52,7 @@ func (qb *performerFilterHandler) validate() error {
careerLength := filter.CareerLength
switch careerLength.Modifier {
case models.CriterionModifierEquals:
start, end, err := utils.ParseYearRangeString(careerLength.Value)
start, end, err := models.ParseYearRangeString(careerLength.Value)
if err != nil {
return fmt.Errorf("invalid career length value: %s", careerLength.Value)
}
@ -70,6 +70,28 @@ func (qb *performerFilterHandler) validate() error {
}
}
// validate date formats
if filter.Birthdate != nil && filter.Birthdate.Value != "" {
if _, err := models.ParseDate(filter.Birthdate.Value); err != nil {
return fmt.Errorf("invalid birthdate value: %s", filter.Birthdate.Value)
}
}
if filter.DeathDate != nil && filter.DeathDate.Value != "" {
if _, err := models.ParseDate(filter.DeathDate.Value); err != nil {
return fmt.Errorf("invalid death date value: %s", filter.DeathDate.Value)
}
}
if filter.CareerStart != nil && filter.CareerStart.Value != "" {
if _, err := models.ParseDate(filter.CareerStart.Value); err != nil {
return fmt.Errorf("invalid career start value: %s", filter.CareerStart.Value)
}
}
if filter.CareerEnd != nil && filter.CareerEnd.Value != "" {
if _, err := models.ParseDate(filter.CareerEnd.Value); err != nil {
return fmt.Errorf("invalid career end value: %s", filter.CareerEnd.Value)
}
}
return nil
}
@ -156,8 +178,8 @@ func (qb *performerFilterHandler) criterionHandler() criterionHandler {
}),
// CareerLength filter is deprecated and non-functional (column removed in schema 78)
intCriterionHandler(filter.CareerStart, tableName+".career_start", nil),
intCriterionHandler(filter.CareerEnd, tableName+".career_end", nil),
&dateCriterionHandler{filter.CareerStart, tableName + ".career_start", nil},
&dateCriterionHandler{filter.CareerEnd, tableName + ".career_end", nil},
stringCriterionHandler(filter.Tattoos, tableName+".tattoos"),
stringCriterionHandler(filter.Piercings, tableName+".piercings"),
intCriterionHandler(filter.Rating100, tableName+".rating", nil),
@ -266,31 +288,39 @@ func convertLegacyCareerLengthFilter(filter *models.PerformerFilterType) {
careerLength := filter.CareerLength
switch careerLength.Modifier {
case models.CriterionModifierEquals:
start, end, _ := utils.ParseYearRangeString(careerLength.Value)
start, end, _ := models.ParseYearRangeString(careerLength.Value)
if start != nil {
filter.CareerStart = &models.IntCriterionInput{
Value: (*start) - 1, // minus one to make it exclusive
start = &models.Date{
Time: start.AddDate(0, 0, -1), // make exclusive
Precision: models.DatePrecisionDay,
}
filter.CareerStart = &models.DateCriterionInput{
Value: start.String(),
Modifier: models.CriterionModifierGreaterThan,
}
}
if end != nil {
filter.CareerEnd = &models.IntCriterionInput{
Value: (*end) + 1, // plus one to make it exclusive
end = &models.Date{
Time: end.AddDate(1, 0, 0), // make exclusive
Precision: models.DatePrecisionDay,
}
filter.CareerEnd = &models.DateCriterionInput{
Value: end.String(), // plus one to make it exclusive
Modifier: models.CriterionModifierLessThan,
}
}
case models.CriterionModifierIsNull:
filter.CareerStart = &models.IntCriterionInput{
filter.CareerStart = &models.DateCriterionInput{
Modifier: models.CriterionModifierIsNull,
}
filter.CareerEnd = &models.IntCriterionInput{
filter.CareerEnd = &models.DateCriterionInput{
Modifier: models.CriterionModifierIsNull,
}
case models.CriterionModifierNotNull:
filter.CareerStart = &models.IntCriterionInput{
filter.CareerStart = &models.DateCriterionInput{
Modifier: models.CriterionModifierNotNull,
}
filter.CareerEnd = &models.IntCriterionInput{
filter.CareerEnd = &models.DateCriterionInput{
Modifier: models.CriterionModifierNotNull,
}
}

70
pkg/sqlite/performer_test.go
View file

@ -65,9 +65,9 @@ func Test_PerformerStore_Create(t *testing.T) {
measurements = "measurements"
fakeTits = "fakeTits"
penisLength = 1.23
circumcised = models.CircumisedEnumCut
careerStart = 2005
careerEnd = 2015
circumcised = models.CircumcisedEnumCut
careerStart = models.DateFromYear(2005)
careerEnd = models.DateFromYear(2015)
tattoos = "tattoos"
piercings = "piercings"
aliases = []string{"alias1", "alias2"}
@ -228,9 +228,9 @@ func Test_PerformerStore_Update(t *testing.T) {
measurements = "measurements"
fakeTits = "fakeTits"
penisLength = 1.23
circumcised = models.CircumisedEnumCut
careerStart = 2005
careerEnd = 2015
circumcised = models.CircumcisedEnumCut
careerStart = models.DateFromYear(2005)
careerEnd = models.DateFromYear(2015)
tattoos = "tattoos"
piercings = "piercings"
aliases = []string{"alias1", "alias2"}
@ -424,8 +424,8 @@ func clearPerformerPartial() models.PerformerPartial {
FakeTits: nullString,
PenisLength: nullFloat,
Circumcised: nullString,
CareerStart: nullInt,
CareerEnd: nullInt,
CareerStart: nullDate,
CareerEnd: nullDate,
Tattoos: nullString,
Piercings: nullString,
Aliases: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet},
@ -457,9 +457,9 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
measurements = "measurements"
fakeTits = "fakeTits"
penisLength = 1.23
circumcised = models.CircumisedEnumCut
careerStart = 2005
careerEnd = 2015
circumcised = models.CircumcisedEnumCut
careerStart = models.DateFromYear(2005)
careerEnd = models.DateFromYear(2015)
tattoos = "tattoos"
piercings = "piercings"
aliases = []string{"alias1", "alias2"}
@ -505,8 +505,8 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) {
FakeTits: models.NewOptionalString(fakeTits),
PenisLength: models.NewOptionalFloat64(penisLength),
Circumcised: models.NewOptionalString(circumcised.String()),
CareerStart: models.NewOptionalInt(careerStart),
CareerEnd: models.NewOptionalInt(careerEnd),
CareerStart: models.NewOptionalDate(careerStart),
CareerEnd: models.NewOptionalDate(careerEnd),
Tattoos: models.NewOptionalString(tattoos),
Piercings: models.NewOptionalString(piercings),
Aliases: &models.UpdateStrings{
@ -1200,7 +1200,7 @@ func TestPerformerQuery(t *testing.T) {
nil,
&models.PerformerFilterType{
Circumcised: &models.CircumcisionCriterionInput{
Value: []models.CircumisedEnum{models.CircumisedEnumCut},
Value: []models.CircumcisedEnum{models.CircumcisedEnumCut},
Modifier: models.CriterionModifierIncludes,
},
},
@ -1213,7 +1213,7 @@ func TestPerformerQuery(t *testing.T) {
nil,
&models.PerformerFilterType{
Circumcised: &models.CircumcisionCriterionInput{
Value: []models.CircumisedEnum{models.CircumisedEnumCut},
Value: []models.CircumcisedEnum{models.CircumcisedEnumCut},
Modifier: models.CriterionModifierExcludes,
},
},
@ -1778,8 +1778,8 @@ func TestPerformerQueryLegacyCareerLength(t *testing.T) {
tests := []struct {
name string
c models.StringCriterionInput
careerStartCrit *models.IntCriterionInput
careerEndCrit *models.IntCriterionInput
careerStartCrit *models.DateCriterionInput
careerEndCrit *models.DateCriterionInput
err bool
}{
{
@ -1788,13 +1788,13 @@ func TestPerformerQueryLegacyCareerLength(t *testing.T) {
Value: value,
Modifier: models.CriterionModifierEquals,
},
careerStartCrit: &models.IntCriterionInput{
Value: 2002,
Modifier: models.CriterionModifierEquals,
careerStartCrit: &models.DateCriterionInput{
Value: "2001-12-31",
Modifier: models.CriterionModifierGreaterThan,
},
careerEndCrit: &models.IntCriterionInput{
Value: 2012,
Modifier: models.CriterionModifierEquals,
careerEndCrit: &models.DateCriterionInput{
Value: "2013-01-01",
Modifier: models.CriterionModifierLessThan,
},
err: false,
},
@ -1811,10 +1811,10 @@ func TestPerformerQueryLegacyCareerLength(t *testing.T) {
c: models.StringCriterionInput{
Modifier: models.CriterionModifierIsNull,
},
careerStartCrit: &models.IntCriterionInput{
careerStartCrit: &models.DateCriterionInput{
Modifier: models.CriterionModifierIsNull,
},
careerEndCrit: &models.IntCriterionInput{
careerEndCrit: &models.DateCriterionInput{
Modifier: models.CriterionModifierIsNull,
},
err: false,
@ -1824,10 +1824,10 @@ func TestPerformerQueryLegacyCareerLength(t *testing.T) {
c: models.StringCriterionInput{
Modifier: models.CriterionModifierNotNull,
},
careerStartCrit: &models.IntCriterionInput{
careerStartCrit: &models.DateCriterionInput{
Modifier: models.CriterionModifierNotNull,
},
careerEndCrit: &models.IntCriterionInput{
careerEndCrit: &models.DateCriterionInput{
Modifier: models.CriterionModifierNotNull,
},
err: false,
@ -1865,16 +1865,16 @@ func TestPerformerQueryLegacyCareerLength(t *testing.T) {
}
for _, performer := range performers {
verifyIntPtr(t, performer.CareerStart, *tt.careerStartCrit)
verifyIntPtr(t, performer.CareerEnd, *tt.careerEndCrit)
verifyDatePtr(t, performer.CareerStart, *tt.careerStartCrit)
verifyDatePtr(t, performer.CareerEnd, *tt.careerEndCrit)
}
})
}
}
func TestPerformerQueryCareerStart(t *testing.T) {
const value = 2002
criterion := models.IntCriterionInput{
const value = "2002"
criterion := models.DateCriterionInput{
Value: value,
Modifier: models.CriterionModifierEquals,
}
@ -1891,7 +1891,7 @@ func TestPerformerQueryCareerStart(t *testing.T) {
}
for _, performer := range performers {
verifyIntPtr(t, performer.CareerStart, criterion)
verifyDatePtr(t, performer.CareerStart, criterion)
}
return nil
@ -1899,8 +1899,8 @@ func TestPerformerQueryCareerStart(t *testing.T) {
}
func TestPerformerQueryCareerEnd(t *testing.T) {
const value = 2012
criterion := models.IntCriterionInput{
const value = "2012"
criterion := models.DateCriterionInput{
Value: value,
Modifier: models.CriterionModifierEquals,
}
@ -1917,7 +1917,7 @@ func TestPerformerQueryCareerEnd(t *testing.T) {
}
for _, performer := range performers {
verifyIntPtr(t, performer.CareerEnd, criterion)
verifyDatePtr(t, performer.CareerEnd, criterion)
}
return nil

27
pkg/sqlite/scene_test.go
View file

@ -2821,6 +2821,33 @@ func verifyIntPtr(t *testing.T, value *int, criterion models.IntCriterionInput)
}
}
func verifyDatePtr(t *testing.T, value *models.Date, criterion models.DateCriterionInput) {
t.Helper()
assert := assert.New(t)
if criterion.Modifier == models.CriterionModifierIsNull {
assert.Nil(value, "expect is null values to be null")
}
if criterion.Modifier == models.CriterionModifierNotNull {
assert.NotNil(value, "expect not null values to be not null")
}
if criterion.Modifier == models.CriterionModifierEquals {
date, _ := models.ParseDate(criterion.Value)
assert.Equal(date, *value)
}
if criterion.Modifier == models.CriterionModifierNotEquals {
date, _ := models.ParseDate(criterion.Value)
assert.NotEqual(date, *value)
}
if criterion.Modifier == models.CriterionModifierGreaterThan {
date, _ := models.ParseDate(criterion.Value)
assert.True(value.After(date))
}
if criterion.Modifier == models.CriterionModifierLessThan {
date, _ := models.ParseDate(criterion.Value)
assert.True(date.After(*value))
}
}
func TestSceneQueryOCounter(t *testing.T) {
const oCounter = 1
oCounterCriterion := models.IntCriterionInput{

Some files were not shown because too many files have changed in this diff Show more