From 1b093e244d40e5c488e3e47f2563f2272b72b078 Mon Sep 17 00:00:00 2001 From: notsafeforgit Date: Sun, 15 Mar 2026 22:17:55 -0700 Subject: [PATCH] fix: update image duplicate checker UI and API handling - Fixes 400 error in ImageDuplicateChecker - Updates UI and frontend types - Fixes tools casing --- pkg/sqlite/image.go | 22 +- pkg/sqlite/image_filter.go | 4 + ui/v2.5/graphql/queries/image.graphql | 2 +- .../ImageDuplicateChecker.tsx | 790 +++++++++++++----- ui/v2.5/src/locales/en-GB.json | 7 +- 5 files changed, 609 insertions(+), 216 deletions(-) diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index 780979270..e106167c7 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -1105,13 +1105,29 @@ func (qb *ImageStore) FindDuplicates(ctx context.Context, distance int) ([][]*mo var hashes []*utils.Phash if err := imageRepository.queryFunc(ctx, query, nil, false, func(rows *sqlx.Rows) error { + var sq struct { + ID int `db:"id"` + Phash *string `db:"phash"` + } + if err := rows.StructScan(&sq); err != nil { + return err + } + + if sq.Phash == nil { + return nil + } + + hashInt, err := utils.StringToPhash(*sq.Phash) + if err != nil { + return nil + } + phash := utils.Phash{ + ID: sq.ID, + Hash: hashInt, Bucket: -1, Duration: -1, } - if err := rows.StructScan(&phash); err != nil { - return err - } hashes = append(hashes, &phash) return nil diff --git a/pkg/sqlite/image_filter.go b/pkg/sqlite/image_filter.go index 4d1d2c4b3..695a8102d 100644 --- a/pkg/sqlite/image_filter.go +++ b/pkg/sqlite/image_filter.go @@ -185,6 +185,10 @@ func (qb *imageFilterHandler) missingCriterionHandler(isMissing *string) criteri case "tags": imageRepository.tags.join(f, "tags_join", "images.id") f.addWhere("tags_join.image_id IS NULL") + case "phash": + f.addInnerJoin("images_files", "", "images_files.image_id = images.id") + f.addLeftJoin(fingerprintTable, "fingerprints_phash", "images_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'") + f.addWhere("fingerprints_phash.fingerprint IS NULL") default: if err := validateIsMissing(*isMissing, []string{ "title", "details", "photographer", "date", "code", "rating", diff --git a/ui/v2.5/graphql/queries/image.graphql b/ui/v2.5/graphql/queries/image.graphql index c74fc4cfd..9ba08c1ee 100644 --- a/ui/v2.5/graphql/queries/image.graphql +++ b/ui/v2.5/graphql/queries/image.graphql @@ -36,7 +36,7 @@ query FindImage($id: ID!, $checksum: String) { } } -query FindDuplicateImages($distance: Int) { +query FindDuplicateImages($distance: Int!) { findDuplicateImages(distance: $distance) { ...ImageData } diff --git a/ui/v2.5/src/components/ImageDuplicateChecker/ImageDuplicateChecker.tsx b/ui/v2.5/src/components/ImageDuplicateChecker/ImageDuplicateChecker.tsx index de6840570..a6f9dc0d0 100644 --- a/ui/v2.5/src/components/ImageDuplicateChecker/ImageDuplicateChecker.tsx +++ b/ui/v2.5/src/components/ImageDuplicateChecker/ImageDuplicateChecker.tsx @@ -2,28 +2,40 @@ import React, { useMemo, useState } from "react"; import { Button, Form, - Spinner, Table, Row, Col, Card, + Dropdown, ButtonGroup, OverlayTrigger, Tooltip, } from "react-bootstrap"; import { FormattedMessage, useIntl } from "react-intl"; +import { Link, useHistory } from "react-router-dom"; +import TextUtils from "src/utils/text"; +import { HoverPopover } from "../Shared/HoverPopover"; +import { TagLink, GalleryLink } from "../Shared/TagLink"; +import { PerformerPopoverButton } from "../Shared/PerformerPopoverButton"; +import { + faFileAlt, + faImages, + faTag, + faBox, + faExclamationTriangle, + faPencilAlt, + faTrash, +} from "@fortawesome/free-solid-svg-icons"; import { useFindDuplicateImagesQuery } from "src/core/generated-graphql"; import * as GQL from "src/core/generated-graphql"; import { PatchContainerComponent } from "src/patch"; -import { LoadingIndicator } from "../Shared/LoadingIndicator"; -import { ErrorMessage } from "../Shared/ErrorMessage"; import { FileSize } from "../Shared/FileSize"; import { Pagination } from "src/components/List/Pagination"; -import { useHistory } from "react-router-dom"; import { DeleteImagesDialog } from "../Images/DeleteImagesDialog"; import { EditImagesDialog } from "../Images/EditImagesDialog"; import { Icon } from "../Shared/Icon"; -import { faPencilAlt, faTrash } from "@fortawesome/free-solid-svg-icons"; + +const CLASSNAME = "duplicate-checker"; const ImageDuplicateCheckerSection = PatchContainerComponent( "ImageDuplicateCheckerSection" @@ -37,8 +49,7 @@ const ImageDuplicateChecker: React.FC = () => { const pageSize = Number.parseInt(query.get("size") ?? "20", 10); const hashDistance = Number.parseInt(query.get("distance") ?? "0", 10); - const [isSearching, setIsSearching] = useState(false); - const [hasSearched, setHasSearched] = useState(false); + const [currentPageSize, setCurrentPageSize] = useState(pageSize); const [checkedImages, setCheckedImages] = useState>( {} ); @@ -47,18 +58,36 @@ const ImageDuplicateChecker: React.FC = () => { const [deletingImages, setDeletingImages] = useState(false); const [editingImages, setEditingImages] = useState(false); - const { data, loading, error, refetch } = useFindDuplicateImagesQuery({ - variables: { distance: hashDistance }, - skip: !hasSearched, - fetchPolicy: "network-only", + const { data: missingPhash } = GQL.useFindImagesQuery({ + variables: { + filter: { + per_page: 0, + }, + image_filter: { + is_missing: "phash", + }, + }, }); - const handleSearch = () => { - setIsSearching(true); - setHasSearched(true); - setCheckedImages({}); - refetch({ distance: hashDistance }).finally(() => setIsSearching(false)); - }; + function maybeRenderMissingPhashWarning() { + const missingPhashes = missingPhash?.findImages.count ?? 0; + if (missingPhashes > 0) { + return ( +

+ + +

+ ); + } + } + + const { data, loading, refetch } = useFindDuplicateImagesQuery({ + variables: { distance: hashDistance }, + fetchPolicy: "network-only", + }); const getGroupTotalSize = (group: GQL.ImageDataFragment[]) => { return group.reduce((groupTotal, img) => { @@ -118,208 +147,551 @@ const ImageDuplicateChecker: React.FC = () => { } }; - if (error) return ; + const pageOptions = useMemo(() => { + const pageSizes = [ + 10, 20, 30, 40, 50, 100, 150, 200, 250, 500, 750, 1000, 1250, 1500, + ]; - const renderGroup = (group: GQL.ImageDataFragment[], index: number) => { - const groupIndex = (currentPage - 1) * pageSize + index + 1; - return ( - - -
Group {groupIndex}
- - Total Size: - - - - - - - - - - - - - - - {group.map((img) => { - const file = img.visual_files[0]; - return ( - - - - - - - - ); - })} - -
ImageDetailsSizeDimensions
- - handleCheck(e.currentTarget.checked, img.id) - } - /> - - {img.title - -
{img.title || "(No Title)"}
-
- {img.visual_files[0]?.path} -
-
ID: {img.id}
- 		- - - {file?.__typename === "ImageFile" || - file?.__typename === "VideoFile" ? ( - <> - {file.width} x {file.height} - - ) : ( - "N/A" - )} -
- - + const filteredSizes = pageSizes.filter((s, i) => { + return ( + allGroups.length > s || i == 0 || allGroups.length > pageSizes[i - 1] + ); + }); + + return filteredSizes.map((size) => { + return ( + + ); + }); + }, [allGroups.length]); + + const setQuery = (q: Record) => { + const newQuery = new URLSearchParams(query); + for (const key of Object.keys(q)) { + const value = q[key]; + if (value !== undefined) { + newQuery.set(key, String(value)); + } else { + newQuery.delete(key); + } + } + history.push({ search: newQuery.toString() }); + }; + + const resetCheckboxSelection = () => { + const updatedImages: Record = {}; + Object.keys(checkedImages).forEach((imageKey) => { + updatedImages[imageKey] = false; + }); + setCheckedImages(updatedImages); + }; + + const findLargestImage = (group: GQL.ImageDataFragment[]) => { + const totalSize = (image: GQL.ImageDataFragment) => { + return image.visual_files.reduce( + (prev: number, f) => Math.max(prev, f.size ?? 0), + 0 + ); + }; + return group.reduce((largest, image) => { + const largestSize = totalSize(largest); + const currentSize = totalSize(image); + return currentSize > largestSize ? image : largest; + }); + }; + + const findLargestResolutionImage = (group: GQL.ImageDataFragment[]) => { + const imgResolution = (image: GQL.ImageDataFragment) => { + return image.visual_files.reduce( + (prev: number, f) => Math.max(prev, (f.height ?? 0) * (f.width ?? 0)), + 0 + ); + }; + return group.reduce((largest, image) => { + const largestSize = imgResolution(largest); + const currentSize = imgResolution(image); + return currentSize > largestSize ? image : largest; + }); + }; + + const findFirstFileByAge = ( + oldest: boolean, + compareImages: GQL.ImageDataFragment[] + ) => { + let selectedFile: GQL.ImageFileDataFragment | GQL.VideoFileDataFragment; + let oldestTimestamp: Date | undefined = undefined; + + for (const file of compareImages.flatMap((s) => s.visual_files)) { + const timestamp: Date = new Date(file.mod_time); + if (oldest) { + if (oldestTimestamp === undefined || timestamp < oldestTimestamp) { + oldestTimestamp = timestamp; + selectedFile = file; + } + } else { + if (oldestTimestamp === undefined || timestamp > oldestTimestamp) { + oldestTimestamp = timestamp; + selectedFile = file; + } + } + } + + return compareImages.find((s) => + s.visual_files.some((f) => f.id === selectedFile?.id) ); }; - return ( -
- - {deletingImages && selectedImages && ( - - )} - {editingImages && selectedImages && ( - - )} - - -

- -

- - + function checkSameResolution(dataGroup: GQL.ImageDataFragment[]) { + const resolutions = dataGroup.map( + (s) => (s.visual_files[0]?.width ?? 0) * (s.visual_files[0]?.height ?? 0) + ); + return new Set(resolutions).size === 1; + } -
- - - - PHash Distance - { - const val = parseInt(e.target.value) || 0; - query.set("distance", val.toString()); - history.push({ search: query.toString() }); + const onSelectLargestClick = () => { + setSelectedImages([]); + const checkedArray: Record = {}; + + pagedGroups.forEach((group) => { + const largest = findLargestImage(group); + group.forEach((image) => { + if (image !== largest) { + checkedArray[image.id] = true; + } + }); + }); + + setCheckedImages(checkedArray); + }; + + const onSelectLargestResolutionClick = () => { + setSelectedImages([]); + const checkedArray: Record = {}; + + pagedGroups.forEach((group) => { + if (checkSameResolution(group)) return; + + const highest = findLargestResolutionImage(group); + group.forEach((image) => { + if (image !== highest) { + checkedArray[image.id] = true; + } + }); + }); + + setCheckedImages(checkedArray); + }; + + const onSelectByAge = (oldest: boolean) => { + setSelectedImages([]); + const checkedArray: Record = {}; + + pagedGroups.forEach((group) => { + const oldestScene = findFirstFileByAge(oldest, group); + group.forEach((image) => { + if (image !== oldestScene) { + checkedArray[image.id] = true; + } + }); + }); + + setCheckedImages(checkedArray); + }; + + const handleDeleteImage = (image: GQL.ImageDataFragment) => { + setSelectedImages([image]); + setDeletingImages(true); + }; + + function renderPagination() { + return ( +
+
+ +
+ {checkCount > 0 && ( + + + {intl.formatMessage({ id: "actions.edit" })} + + } + > + + + + {intl.formatMessage({ id: "actions.delete" })} + + } + > + + + + )} + { + setQuery({ page: newPage === 1 ? undefined : newPage }); + resetCheckboxSelection(); + }} + /> + { + setCurrentPageSize(parseInt(e.currentTarget.value, 10)); + setQuery({ + size: + e.currentTarget.value === "20" + ? undefined + : e.currentTarget.value, + }); + resetCheckboxSelection(); + }} + > + {pageOptions} + +
+ ); + } + + function maybeRenderPopoverButtonGroup(image: GQL.ImageDataFragment) { + if ( + image.tags.length > 0 || + image.performers.length > 0 || + image.galleries.length > 0 || + image.visual_files.length > 1 || + image.organized + ) { + return ( + + {image.tags.length > 0 && ( + ( + + ))} + > + + + )} + {image.performers.length > 0 && ( + + )} + {image.galleries.length > 0 && ( + ( + + ))} + > + + + )} + {image.visual_files.length > 1 && ( + - - 0 = exact matches. - - - - - - - -
+ + )} + {image.organized && ( +
+ +
+ )} + + ); + } + } - {loading && } - - {hasSearched && !loading && !error && allGroups.length === 0 && ( -
-

- No duplicates found with the current distance. -

-
- )} - - {hasSearched && !loading && !error && allGroups.length > 0 && ( -
-
- Found {allGroups.length} duplicate groups -
- {checkCount > 0 && ( - - - {intl.formatMessage({ id: "actions.edit" })} - - } - > - - - - {intl.formatMessage({ id: "actions.delete" })} - - } - > - - - - )} -
- )} - - {pagedGroups.map((group, index) => renderGroup(group, index))} - - {allGroups.length > pageSize && ( -
- { - query.set("page", page.toString()); - history.push({ search: query.toString() }); - }} + return ( + +
+ + {deletingImages && selectedImages && ( + -
- )} - -
+ )} + {editingImages && selectedImages && ( + + )} + +

+ +

+ +
+ + + + + + + + setQuery({ + distance: + e.currentTarget.value === "0" + ? undefined + : e.currentTarget.value, + page: undefined, + }) + } + defaultValue={hashDistance} + className="input-control ml-4" + > + + + + + + + + + + + + + + + + + + + + + resetCheckboxSelection()}> + {intl.formatMessage({ id: "dupe_check.select_none" })} + + + onSelectLargestResolutionClick()} + > + {intl.formatMessage({ + id: "dupe_check.select_all_but_largest_resolution", + })} + + + onSelectLargestClick()}> + {intl.formatMessage({ + id: "dupe_check.select_all_but_largest_file", + })} + + + onSelectByAge(true)}> + {intl.formatMessage({ + id: "dupe_check.select_oldest", + })} + + + onSelectByAge(false)}> + {intl.formatMessage({ + id: "dupe_check.select_youngest", + })} + + + + + + +
+ + {maybeRenderMissingPhashWarning()} + {renderPagination()} + + + + + + + + + + + + + + + + + + + + + + + + {pagedGroups.map((group, groupIndex) => + group.map((image, i) => { + const file = image.visual_files[0]; + + return ( + + {i === 0 && groupIndex !== 0 ? ( + + ) : undefined} + + + + + + + + + + + ); + }) + )} + +
{intl.formatMessage({ id: "details" })} {intl.formatMessage({ id: "filesize" })}{intl.formatMessage({ id: "resolution" })}{intl.formatMessage({ id: "actions.delete" })}
+ + handleCheck(e.currentTarget.checked, image.id) + } + /> + + + } + placement="right" + > + + + +

+ + {image.title || + TextUtils.fileNameFromPath(file?.path ?? "")} + +

+

{file?.path ?? ""}

+ 		+ {maybeRenderPopoverButtonGroup(image)} + + + + {file?.__typename === "ImageFile" || + file?.__typename === "VideoFile" ? ( + <> + {file.width ?? 0}x{file.height ?? 0} + + ) : ( + "N/A" + )} + + +
+ + {allGroups.length === 0 && !loading && ( +

No duplicates found.

+ )} + + {loading && ( +
+ +

Loading...

+
+ )} + + {renderPagination()} + +
+ ); }; diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index 5f6fb1158..ee41130a8 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -625,9 +625,9 @@ "set_name_date_details_from_metadata_if_present": "Set name, date, details from embedded file metadata" }, "tools": { - "graphql_playground": "GraphQL playground", + "graphql_playground": "GraphQL Playground", "heading": "Tools", - "scene_duplicate_checker": "Scene duplicate checker", + "scene_duplicate_checker": "Scene Duplicate Checker", "scene_filename_parser": { "add_field": "Add Field", "capitalize_title": "Capitalize title", @@ -639,7 +639,7 @@ "ignored_words": "Ignored words", "matches_with": "Matches with {i}", "select_parser_recipe": "Select Parser Recipe", - "title": "Scene filename parser", + "title": "Scene Filename Parser", "whitespace_chars": "Whitespace characters", "whitespace_chars_desc": "These characters will be replaced with whitespace in the title" }, @@ -1120,6 +1120,7 @@ "distance": "Distance", "donate": "Donate", "dupe_check": { + "missing_phash_warning": "Missing phashes for {count} images. Please run the phash generation task.", "description": "Levels below 'Exact' can take longer to calculate. False positives might also be returned on lower accuracy levels.", "duration_diff": "Maximum Duration Difference", "duration_options": {