diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index 7f07e4579..ae356e468 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -53,6 +53,9 @@ type Query { duration_diff: Float ): [[Scene!]!]! + "Find duplicate images" + findDuplicateImages(distance: Int! = 0): [[Image!]!]! + "Return valid stream paths" sceneStreams(id: ID): [SceneStreamEndpoint!]! diff --git a/internal/api/resolver_query_find_image.go b/internal/api/resolver_query_find_image.go index 90eaf33c0..f547151b1 100644 --- a/internal/api/resolver_query_find_image.go +++ b/internal/api/resolver_query_find_image.go @@ -134,3 +134,14 @@ func (r *queryResolver) AllImages(ctx context.Context) (ret []*models.Image, err return ret, nil } + +func (r *queryResolver) FindDuplicateImages(ctx context.Context, distance int) (ret [][]*models.Image, err error) { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + ret, err = r.repository.Image.FindDuplicates(ctx, distance) + return err + }); err != nil { + return nil, err + } + + return ret, nil +} diff --git a/pkg/models/mocks/ImageReaderWriter.go b/pkg/models/mocks/ImageReaderWriter.go index f2c9934be..f3f05aaff 100644 --- a/pkg/models/mocks/ImageReaderWriter.go +++ b/pkg/models/mocks/ImageReaderWriter.go @@ -370,6 +370,29 @@ func (_m *ImageReaderWriter) FindByZipFileID(ctx context.Context, zipFileID mode return r0, r1 } +// FindDuplicates provides a mock function with given fields: ctx, distance +func (_m *ImageReaderWriter) FindDuplicates(ctx context.Context, distance int) ([][]*models.Image, error) { + ret := _m.Called(ctx, distance) + + var r0 [][]*models.Image + if rf, ok := ret.Get(0).(func(context.Context, int) [][]*models.Image); ok { + r0 = rf(ctx, distance) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([][]*models.Image) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int) error); ok { + r1 = rf(ctx, distance) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // FindMany provides a mock function with given fields: ctx, ids func (_m *ImageReaderWriter) FindMany(ctx context.Context, ids []int) ([]*models.Image, error) { ret := _m.Called(ctx, ids) diff --git a/pkg/models/repository_image.go b/pkg/models/repository_image.go index 99dab3479..10e0d195a 100644 --- a/pkg/models/repository_image.go +++ b/pkg/models/repository_image.go @@ -19,6 +19,7 @@ type ImageFinder interface { FindByZipFileID(ctx context.Context, zipFileID FileID) ([]*Image, error) FindByGalleryID(ctx context.Context, galleryID int) ([]*Image, error) FindByGalleryIDIndex(ctx context.Context, galleryID int, index uint) (*Image, error) + FindDuplicates(ctx context.Context, distance int) ([][]*Image, error) } // ImageQueryer provides methods to query images. diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index 4d9ebad1b..1ac8a5dca 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -6,11 +6,13 @@ import ( "errors" "fmt" "path/filepath" - "slices" + "strconv" + "strings" "github.com/jmoiron/sqlx" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/sliceutil" + "github.com/stashapp/stash/pkg/utils" "gopkg.in/guregu/null.v4" "gopkg.in/guregu/null.v4/zero" @@ -409,6 +411,11 @@ func (qb *ImageStore) Find(ctx context.Context, id int) (*models.Image, error) { func (qb *ImageStore) FindMany(ctx context.Context, ids []int) ([]*models.Image, error) { images := make([]*models.Image, len(ids)) + idToIndex := make(map[int]int, len(ids)) + for i, id := range ids { + idToIndex[id] = i + } + if err := batchExec(ids, defaultBatchSize, func(batch []int) error { q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(batch)) unsorted, err := qb.getMany(ctx, q) @@ -417,8 +424,9 @@ func (qb *ImageStore) FindMany(ctx context.Context, ids []int) ([]*models.Image, } for _, s := range unsorted { - i := slices.Index(ids, s.ID) - images[i] = s + if i, ok := idToIndex[s.ID]; ok { + images[i] = s + } } return nil @@ -837,7 +845,7 @@ func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFi ) filepathColumn := "folders.path || '" + string(filepath.Separator) + "' || files.basename" - searchColumns := []string{"images.title", "images.details", filepathColumn, "files_fingerprints.fingerprint"} + searchColumns := []string{"images.title", filepathColumn, "files_fingerprints.fingerprint"} query.parseQueryString(searchColumns, *q) } @@ -1093,3 +1101,94 @@ func (qb *ImageStore) UpdateTags(ctx context.Context, imageID int, tagIDs []int) func (qb *ImageStore) GetURLs(ctx context.Context, imageID int) ([]string, error) { return imagesURLsTableMgr.get(ctx, imageID) } + +var findExactImageDuplicateQuery = ` +SELECT GROUP_CONCAT(DISTINCT image_id) as ids +FROM ( + SELECT images_files.image_id + , files.size as file_size + , files_fingerprints.fingerprint as phash + FROM images_files + JOIN files ON images_files.file_id = files.id + JOIN files_fingerprints ON images_files.file_id = files_fingerprints.file_id + WHERE files_fingerprints.type = 'phash' + AND files_fingerprints.fingerprint != zeroblob(8) + AND files_fingerprints.fingerprint != '' +) +GROUP BY phash +HAVING COUNT(DISTINCT image_id) > 1 +ORDER BY SUM(file_size) DESC; +` + +func (qb *ImageStore) FindDuplicates(ctx context.Context, distance int) ([][]*models.Image, error) { + var dupeIds [][]int + if distance == 0 { + var ids []string + if err := dbWrapper.Select(ctx, &ids, findExactImageDuplicateQuery); err != nil { + return nil, err + } + + for _, id := range ids { + strIds := strings.Split(id, ",") + var imageIds []int + for _, strId := range strIds { + if intId, err := strconv.Atoi(strId); err == nil { + imageIds = sliceutil.AppendUnique(imageIds, intId) + } + } + // filter out + if len(imageIds) > 1 { + dupeIds = append(dupeIds, imageIds) + } + } + } else { + query := ` + SELECT images.id, files_fingerprints.fingerprint as phash + FROM images + JOIN images_files ON images.id = images_files.image_id + JOIN files_fingerprints ON images_files.file_id = files_fingerprints.file_id + WHERE files_fingerprints.type = 'phash'` + + var hashes []*utils.Phash + if err := imageRepository.queryFunc(ctx, query, nil, false, func(rows *sqlx.Rows) error { + phash := utils.Phash{ + Bucket: -1, + Duration: -1, + } + if err := rows.StructScan(&phash); err != nil { + return err + } + + hashes = append(hashes, &phash) + return nil + }); err != nil { + return nil, err + } + + dupeIds = utils.FindDuplicates(hashes, distance, -1) + } + + var allIds []int + for _, comp := range dupeIds { + allIds = append(allIds, comp...) + } + + if len(allIds) == 0 { + return nil, nil + } + + allImages, err := qb.FindMany(ctx, allIds) + if err != nil { + return nil, err + } + + var result [][]*models.Image + offset := 0 + for _, comp := range dupeIds { + group := allImages[offset : offset+len(comp)] + result = append(result, group) + offset += len(comp) + } + + return result, nil +} diff --git a/pkg/sqlite/image_filter.go b/pkg/sqlite/image_filter.go index a7351e52e..ccb11973a 100644 --- a/pkg/sqlite/image_filter.go +++ b/pkg/sqlite/image_filter.go @@ -189,6 +189,10 @@ func (qb *imageFilterHandler) missingCriterionHandler(isMissing *string) criteri case "tags": imageRepository.tags.leftJoin(f, "tags_join", "images.id") f.addWhere("tags_join.image_id IS NULL") + case "phash": + f.addInnerJoin("images_files", "", "images_files.image_id = images.id") + f.addLeftJoin(fingerprintTable, "fingerprints_phash", "images_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'") + f.addWhere("fingerprints_phash.fingerprint IS NULL") default: if err := validateIsMissing(*isMissing, []string{ "title", "details", "photographer", "date", "code", "rating", diff --git a/pkg/sqlite/image_test.go b/pkg/sqlite/image_test.go index 85337c911..3bad40b3b 100644 --- a/pkg/sqlite/image_test.go +++ b/pkg/sqlite/image_test.go @@ -1596,20 +1596,6 @@ func TestImageQueryQ(t *testing.T) { }) } -func TestImageQueryQ_Details(t *testing.T) { - withTxn(func(ctx context.Context) error { - const imageIdx = 3 - - q := getImageStringValue(imageIdx, detailsField) - - sqb := db.Image - - imageQueryQ(ctx, t, sqb, q, imageIdx) - - return nil - }) -} - func queryImagesWithCount(ctx context.Context, sqb models.ImageReader, imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) ([]*models.Image, int, error) { result, err := sqb.Query(ctx, models.ImageQueryOptions{ QueryOptions: models.QueryOptions{ diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index c2093431d..e1f750477 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "path/filepath" - "slices" "sort" "strconv" "strings" @@ -533,9 +532,15 @@ func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene, return nil, err } + idToIndex := make(map[int]int, len(ids)) + for i, id := range ids { + idToIndex[id] = i + } + for _, s := range unsorted { - i := slices.Index(ids, s.ID) - scenes[i] = s + if i, ok := idToIndex[s.ID]; ok { + scenes[i] = s + } } for i := range scenes { @@ -1472,11 +1477,26 @@ func (qb *SceneStore) FindDuplicates(ctx context.Context, distance int, duration dupeIds = utils.FindDuplicates(hashes, distance, durationDiff) } + var allIds []int + for _, comp := range dupeIds { + allIds = append(allIds, comp...) + } + + if len(allIds) == 0 { + return nil, nil + } + + allScenes, err := qb.FindMany(ctx, allIds) + if err != nil { + return nil, err + } + var duplicates [][]*models.Scene - for _, sceneIds := range dupeIds { - if scenes, err := qb.FindMany(ctx, sceneIds); err == nil { - duplicates = append(duplicates, scenes) - } + offset := 0 + for _, comp := range dupeIds { + group := allScenes[offset : offset+len(comp)] + duplicates = append(duplicates, group) + offset += len(comp) } sortByPath(duplicates) diff --git a/pkg/utils/phash.go b/pkg/utils/phash.go index 413293c65..5ca72e4fb 100644 --- a/pkg/utils/phash.go +++ b/pkg/utils/phash.go @@ -2,14 +2,16 @@ package utils import ( "math" + "math/bits" + "runtime" "strconv" + "sync" - "github.com/corona10/goimagehash" "github.com/stashapp/stash/pkg/sliceutil" ) type Phash struct { - SceneID int `db:"id"` + ID int `db:"id"` Hash int64 `db:"phash"` Duration float64 `db:"duration"` Neighbors []int @@ -17,35 +19,65 @@ type Phash struct { } func FindDuplicates(hashes []*Phash, distance int, durationDiff float64) [][]int { - for i, scene := range hashes { - sceneHash := goimagehash.NewImageHash(uint64(scene.Hash), goimagehash.PHash) - for j, neighbor := range hashes { - if i != j && scene.SceneID != neighbor.SceneID { - neighbourDurationDistance := 0. - if scene.Duration > 0 && neighbor.Duration > 0 { - neighbourDurationDistance = math.Abs(scene.Duration - neighbor.Duration) - } - if (neighbourDurationDistance <= durationDiff) || (durationDiff < 0) { - neighborHash := goimagehash.NewImageHash(uint64(neighbor.Hash), goimagehash.PHash) - neighborDistance, _ := sceneHash.Distance(neighborHash) - if neighborDistance <= distance { - scene.Neighbors = append(scene.Neighbors, j) + // Pre-calculate hash values to avoid allocations and method calls in the inner loop + uintHashes := make([]uint64, len(hashes)) + for i, h := range hashes { + uintHashes[i] = uint64(h.Hash) + } + + numHashes := len(hashes) + numWorkers := runtime.GOMAXPROCS(0) + var wg sync.WaitGroup + wg.Add(numWorkers) + + // Distribute work among workers + for w := 0; w < numWorkers; w++ { + go func(workerID int) { + defer wg.Done() + for i := workerID; i < numHashes; i += numWorkers { + subject := hashes[i] + subjectHash := uintHashes[i] + + for j := 0; j < numHashes; j++ { + if i == j { + continue + } + neighbor := hashes[j] + if subject.ID == neighbor.ID { + continue + } + + // Check duration if applicable (for scenes) + if durationDiff >= 0 { + if subject.Duration > 0 && neighbor.Duration > 0 { + if math.Abs(subject.Duration-neighbor.Duration) > durationDiff { + continue + } + } + } + + neighborHash := uintHashes[j] + // Hamming distance using native bit counting + if bits.OnesCount64(subjectHash^neighborHash) <= distance { + subject.Neighbors = append(subject.Neighbors, j) } } } - } + }(w) } - var buckets [][]int - for _, scene := range hashes { - if len(scene.Neighbors) > 0 && scene.Bucket == -1 { - bucket := len(buckets) - scenes := []int{scene.SceneID} - scene.Bucket = bucket - findNeighbors(bucket, scene.Neighbors, hashes, &scenes) + wg.Wait() - if len(scenes) > 1 { - buckets = append(buckets, scenes) + var buckets [][]int + for _, subject := range hashes { + if len(subject.Neighbors) > 0 && subject.Bucket == -1 { + bucket := len(buckets) + ids := []int{subject.ID} + subject.Bucket = bucket + findNeighbors(bucket, subject.Neighbors, hashes, &ids) + + if len(ids) > 1 { + buckets = append(buckets, ids) } } } @@ -53,13 +85,13 @@ func FindDuplicates(hashes []*Phash, distance int, durationDiff float64) [][]int return buckets } -func findNeighbors(bucket int, neighbors []int, hashes []*Phash, scenes *[]int) { +func findNeighbors(bucket int, neighbors []int, hashes []*Phash, ids *[]int) { for _, id := range neighbors { hash := hashes[id] if hash.Bucket == -1 { hash.Bucket = bucket - *scenes = sliceutil.AppendUnique(*scenes, hash.SceneID) - findNeighbors(bucket, hash.Neighbors, hashes, scenes) + *ids = sliceutil.AppendUnique(*ids, hash.ID) + findNeighbors(bucket, hash.Neighbors, hashes, ids) } } } diff --git a/ui/v2.5/graphql/queries/image.graphql b/ui/v2.5/graphql/queries/image.graphql index d2c6cdac8..41df8b67e 100644 --- a/ui/v2.5/graphql/queries/image.graphql +++ b/ui/v2.5/graphql/queries/image.graphql @@ -35,3 +35,9 @@ query FindImage($id: ID!, $checksum: String) { ...ImageData } } + +query FindDuplicateImages($distance: Int!) { + findDuplicateImages(distance: $distance) { + ...SlimImageData + } +} diff --git a/ui/v2.5/src/App.tsx b/ui/v2.5/src/App.tsx index d08274b18..9bb40e7cb 100644 --- a/ui/v2.5/src/App.tsx +++ b/ui/v2.5/src/App.tsx @@ -82,6 +82,9 @@ const SceneFilenameParser = lazyComponent( const SceneDuplicateChecker = lazyComponent( () => import("./components/SceneDuplicateChecker/SceneDuplicateChecker") ); +const ImageDuplicateChecker = lazyComponent( + () => import("./components/ImageDuplicateChecker/ImageDuplicateChecker") +); const appleRendering = isPlatformUniquelyRenderedByApple(); @@ -269,6 +272,10 @@ export const App: React.FC = () => { path="/sceneDuplicateChecker" component={SceneDuplicateChecker} /> + diff --git a/ui/v2.5/src/components/ImageDuplicateChecker/ImageDuplicateChecker.tsx b/ui/v2.5/src/components/ImageDuplicateChecker/ImageDuplicateChecker.tsx new file mode 100644 index 000000000..22f4deead --- /dev/null +++ b/ui/v2.5/src/components/ImageDuplicateChecker/ImageDuplicateChecker.tsx @@ -0,0 +1,712 @@ +import React, { useMemo, useState } from "react"; +import { + Button, + Form, + Table, + Row, + Col, + Card, + Dropdown, + ButtonGroup, + OverlayTrigger, + Tooltip, +} from "react-bootstrap"; +import { FormattedMessage, useIntl } from "react-intl"; +import { Link, useHistory } from "react-router-dom"; +import TextUtils from "src/utils/text"; +import { HoverPopover } from "../Shared/HoverPopover"; +import { TagLink, GalleryLink } from "../Shared/TagLink"; +import { PerformerPopoverButton } from "../Shared/PerformerPopoverButton"; +import { + faFileAlt, + faImages, + faTag, + faBox, + faExclamationTriangle, + faPencilAlt, + faTrash, +} from "@fortawesome/free-solid-svg-icons"; +import { useFindDuplicateImagesQuery } from "src/core/generated-graphql"; +import * as GQL from "src/core/generated-graphql"; +import { PatchContainerComponent } from "src/patch"; +import { FileSize } from "../Shared/FileSize"; +import { Pagination } from "src/components/List/Pagination"; +import { DeleteImagesDialog } from "../Images/DeleteImagesDialog"; +import { EditImagesDialog } from "../Images/EditImagesDialog"; +import { Icon } from "../Shared/Icon"; +import { LoadingIndicator } from "../Shared/LoadingIndicator"; +import { ErrorMessage } from "../Shared/ErrorMessage"; + +const CLASSNAME = "duplicate-checker"; + +const ImageDuplicateCheckerSection = PatchContainerComponent( + "ImageDuplicateCheckerSection" +); + +const ImageDuplicateChecker: React.FC = () => { + const intl = useIntl(); + const history = useHistory(); + const query = new URLSearchParams(history.location.search); + const currentPage = Number.parseInt(query.get("page") ?? "1", 10); + const pageSize = Number.parseInt(query.get("size") ?? "20", 10); + const hashDistance = Number.parseInt(query.get("distance") ?? "0", 10); + + const [currentPageSize, setCurrentPageSize] = useState(pageSize); + const [checkedImages, setCheckedImages] = useState>( + {} + ); + const [selectedImages, setSelectedImages] = + useState(); + const [deletingImages, setDeletingImages] = useState(false); + const [editingImages, setEditingImages] = useState(false); + + const { data: missingPhash } = GQL.useFindImagesQuery({ + variables: { + filter: { + per_page: 0, + }, + image_filter: { + is_missing: "phash", + }, + }, + }); + + function maybeRenderMissingPhashWarning() { + const missingPhashes = missingPhash?.findImages.count ?? 0; + if (missingPhashes > 0) { + return ( +

+ + +

+ ); + } + } + + const { data, loading, refetch } = useFindDuplicateImagesQuery({ + variables: { distance: hashDistance }, + fetchPolicy: "no-cache", + }); + + const getGroupTotalSize = (group: GQL.SlimImageDataFragment[]) => { + return group.reduce((groupTotal, img) => { + const imgTotal = img.visual_files.reduce( + (fileTotal, file) => fileTotal + (file.size ?? 0), + 0 + ); + return groupTotal + imgTotal; + }, 0); + }; + + const allGroups = useMemo(() => { + const groups = data?.findDuplicateImages ?? []; + + const groupSizes = new Map(); + groups.forEach((group) => { + groupSizes.set(group, getGroupTotalSize(group)); + }); + + return [...groups].sort((a, b) => { + return (groupSizes.get(b) ?? 0) - (groupSizes.get(a) ?? 0); + }); + }, [data?.findDuplicateImages]); + + const pagedGroups = useMemo(() => { + const start = (currentPage - 1) * pageSize; + return allGroups.slice(start, start + pageSize); + }, [allGroups, currentPage, pageSize]); + + const checkCount = Object.keys(checkedImages).filter( + (id) => checkedImages[id] + ).length; + + const handleCheck = (checked: boolean, imageID: string) => { + setCheckedImages({ ...checkedImages, [imageID]: checked }); + }; + + const handleDeleteChecked = () => { + setSelectedImages(allGroups.flat().filter((i) => checkedImages[i.id])); + setDeletingImages(true); + }; + + const onEdit = () => { + setSelectedImages(allGroups.flat().filter((i) => checkedImages[i.id])); + setEditingImages(true); + setCheckedImages({}); + }; + + const onDeleteDialogClosed = async (confirmed: boolean) => { + setDeletingImages(false); + setSelectedImages(undefined); + if (confirmed) { + setCheckedImages({}); + await refetch(); + } + }; + + const onEditDialogClosed = async (applied: boolean) => { + setEditingImages(false); + setSelectedImages(undefined); + if (applied) { + await refetch(); + } + }; + + const pageOptions = useMemo(() => { + const pageSizes = [ + 10, 20, 30, 40, 50, 100, 150, 200, 250, 500, 750, 1000, 1250, 1500, + ]; + + const filteredSizes = pageSizes.filter((s, i) => { + return ( + allGroups.length > s || i == 0 || allGroups.length > pageSizes[i - 1] + ); + }); + + return filteredSizes.map((size) => { + return ( + + ); + }); + }, [allGroups.length]); + + if (loading) return ; + if (!data) return ; + + const setQuery = (q: Record) => { + const newQuery = new URLSearchParams(query); + for (const key of Object.keys(q)) { + const value = q[key]; + if (value !== undefined) { + newQuery.set(key, String(value)); + } else { + newQuery.delete(key); + } + } + history.push({ search: newQuery.toString() }); + }; + + const resetCheckboxSelection = () => { + const updatedImages: Record = {}; + Object.keys(checkedImages).forEach((imageKey) => { + updatedImages[imageKey] = false; + }); + setCheckedImages(updatedImages); + }; + + const findLargestImage = (group: GQL.SlimImageDataFragment[]) => { + const totalSize = (image: GQL.SlimImageDataFragment) => { + return image.visual_files.reduce( + (prev: number, f) => Math.max(prev, f.size ?? 0), + 0 + ); + }; + return group.reduce((largest, image) => { + const largestSize = totalSize(largest); + const currentSize = totalSize(image); + return currentSize > largestSize ? image : largest; + }); + }; + + const findLargestResolutionImage = (group: GQL.SlimImageDataFragment[]) => { + const imgResolution = (image: GQL.SlimImageDataFragment) => { + return image.visual_files.reduce( + (prev: number, f) => Math.max(prev, (f.height ?? 0) * (f.width ?? 0)), + 0 + ); + }; + return group.reduce((largest, image) => { + const largestSize = imgResolution(largest); + const currentSize = imgResolution(image); + return currentSize > largestSize ? image : largest; + }); + }; + + const findFirstFileByAge = ( + oldest: boolean, + compareImages: GQL.SlimImageDataFragment[] + ) => { + let selectedFile: GQL.ImageFileDataFragment | GQL.VideoFileDataFragment; + let oldestTimestamp: Date | undefined = undefined; + + for (const file of compareImages.flatMap((s) => s.visual_files)) { + const timestamp: Date = new Date(file.mod_time); + if (oldest) { + if (oldestTimestamp === undefined || timestamp < oldestTimestamp) { + oldestTimestamp = timestamp; + selectedFile = file; + } + } else { + if (oldestTimestamp === undefined || timestamp > oldestTimestamp) { + oldestTimestamp = timestamp; + selectedFile = file; + } + } + } + + return compareImages.find((s) => + s.visual_files.some((f) => f.id === selectedFile?.id) + ); + }; + + function checkSameResolution(dataGroup: GQL.SlimImageDataFragment[]) { + const resolutions = dataGroup.map((s) => { + return s.visual_files.reduce( + (prev, f) => Math.max(prev, (f.height ?? 0) * (f.width ?? 0)), + 0 + ); + }); + return new Set(resolutions).size === 1; + } + + const onSelectLargestClick = () => { + setSelectedImages([]); + const checkedArray: Record = {}; + + pagedGroups.forEach((group) => { + const largest = findLargestImage(group); + group.forEach((image) => { + if (image !== largest) { + checkedArray[image.id] = true; + } + }); + }); + + setCheckedImages(checkedArray); + }; + + const onSelectLargestResolutionClick = () => { + setSelectedImages([]); + const checkedArray: Record = {}; + + pagedGroups.forEach((group) => { + if (checkSameResolution(group)) return; + + const highest = findLargestResolutionImage(group); + group.forEach((image) => { + if (image !== highest) { + checkedArray[image.id] = true; + } + }); + }); + + setCheckedImages(checkedArray); + }; + + const onSelectByAge = (oldest: boolean) => { + setSelectedImages([]); + const checkedArray: Record = {}; + + pagedGroups.forEach((group) => { + const oldestScene = findFirstFileByAge(oldest, group); + group.forEach((image) => { + if (image !== oldestScene) { + checkedArray[image.id] = true; + } + }); + }); + + setCheckedImages(checkedArray); + }; + + const handleDeleteImage = (image: GQL.SlimImageDataFragment) => { + setSelectedImages([image]); + setDeletingImages(true); + }; + + function renderPagination() { + return ( +
+
+ +
+ {checkCount > 0 && ( + + + {intl.formatMessage({ id: "actions.edit" })} + + } + > + + + + {intl.formatMessage({ id: "actions.delete" })} + + } + > + + + + )} + { + setQuery({ page: newPage === 1 ? undefined : newPage }); + resetCheckboxSelection(); + }} + /> + { + setCurrentPageSize(parseInt(e.currentTarget.value, 10)); + setQuery({ + size: + e.currentTarget.value === "20" + ? undefined + : e.currentTarget.value, + }); + resetCheckboxSelection(); + }} + > + {pageOptions} + +
+ ); + } + + function maybeRenderPopoverButtonGroup(image: GQL.SlimImageDataFragment) { + if ( + image.tags.length > 0 || + image.performers.length > 0 || + image.galleries.length > 0 || + image.visual_files.length > 1 || + image.organized + ) { + return ( + + {image.tags.length > 0 && ( + ( + + ))} + > + + + )} + {image.performers.length > 0 && ( + + )} + {image.galleries.length > 0 && ( + ( + + ))} + > + + + )} + {image.visual_files.length > 1 && ( + + } + > + + + )} + {image.organized && ( +
+ +
+ )} + + ); + } + } + + return ( + +
+ + {deletingImages && selectedImages && ( + + )} + {editingImages && selectedImages && ( + + )} + +

+ +

+ +
+ + + + + + + + setQuery({ + distance: + e.currentTarget.value === "0" + ? undefined + : e.currentTarget.value, + page: undefined, + }) + } + defaultValue={hashDistance} + className="input-control ml-4" + > + + + + + + + + + + + + + + + + + + + + + resetCheckboxSelection()}> + {intl.formatMessage({ id: "dupe_check.select_none" })} + + + onSelectLargestResolutionClick()} + > + {intl.formatMessage({ + id: "dupe_check.select_all_but_largest_resolution", + })} + + + onSelectLargestClick()}> + {intl.formatMessage({ + id: "dupe_check.select_all_but_largest_file", + })} + + + onSelectByAge(true)}> + {intl.formatMessage({ + id: "dupe_check.select_oldest", + })} + + + onSelectByAge(false)}> + {intl.formatMessage({ + id: "dupe_check.select_youngest", + })} + + + + + + +
+ + {maybeRenderMissingPhashWarning()} + {renderPagination()} + + + + + + + + + + + + + + + + + + + + + + + + {pagedGroups.map((group, groupIndex) => + group.map((image, i) => { + const file = image.visual_files[0]; + + return ( + + {i === 0 && groupIndex !== 0 ? ( + + ) : undefined} + + + + + + + + + + + ); + }) + )} + +
{intl.formatMessage({ id: "details" })} {intl.formatMessage({ id: "filesize" })}{intl.formatMessage({ id: "resolution" })}{intl.formatMessage({ id: "actions.delete" })}
+ + handleCheck(e.currentTarget.checked, image.id) + } + /> + + + } + placement="right" + > + + + +

+ + {image.title || + TextUtils.fileNameFromPath(file?.path ?? "")} + +

+

{file?.path ?? ""}

+ 		+ {maybeRenderPopoverButtonGroup(image)} + + + + {file?.__typename === "ImageFile" || + file?.__typename === "VideoFile" ? ( + <> + {file.width ?? 0}x{file.height ?? 0} + + ) : ( + "N/A" + )} + + +
+ + {allGroups.length === 0 && !loading && ( +

No duplicates found.

+ )} + + {loading && ( +
+ +

Loading...

+
+ )} + + {renderPagination()} + +
+ + ); +}; + +export default ImageDuplicateChecker; diff --git a/ui/v2.5/src/components/Images/DeleteImagesDialog.tsx b/ui/v2.5/src/components/Images/DeleteImagesDialog.tsx index d57c60ab4..45d683682 100644 --- a/ui/v2.5/src/components/Images/DeleteImagesDialog.tsx +++ b/ui/v2.5/src/components/Images/DeleteImagesDialog.tsx @@ -61,11 +61,12 @@ export const DeleteImagesDialog: React.FC = ( try { await deleteImage(); Toast.success(toastMessage); + props.onClose(true); } catch (e) { Toast.error(e); + props.onClose(false); } setIsDeleting(false); - props.onClose(true); } function maybeRenderDeleteFileAlert() { diff --git a/ui/v2.5/src/components/SceneDuplicateChecker/SceneDuplicateChecker.tsx b/ui/v2.5/src/components/SceneDuplicateChecker/SceneDuplicateChecker.tsx index d396a01f4..b226886c1 100644 --- a/ui/v2.5/src/components/SceneDuplicateChecker/SceneDuplicateChecker.tsx +++ b/ui/v2.5/src/components/SceneDuplicateChecker/SceneDuplicateChecker.tsx @@ -92,9 +92,15 @@ export const SceneDuplicateChecker: React.FC = () => { const scenes = useMemo(() => { const groups = data?.findDuplicateScenes ?? []; + + const groupSizes = new Map(); + groups.forEach((group) => { + groupSizes.set(group, getGroupTotalSize(group)); + }); + // Sort by total file size descending (largest groups first) return [...groups].sort((a, b) => { - return getGroupTotalSize(b) - getGroupTotalSize(a); + return (groupSizes.get(b) ?? 0) - (groupSizes.get(a) ?? 0); }); }, [data?.findDuplicateScenes]); diff --git a/ui/v2.5/src/components/Settings/SettingsToolsPanel.tsx b/ui/v2.5/src/components/Settings/SettingsToolsPanel.tsx index e3577a499..5e7f6bee5 100644 --- a/ui/v2.5/src/components/Settings/SettingsToolsPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsToolsPanel.tsx @@ -48,6 +48,20 @@ export const SettingsToolsPanel: React.FC = () => { /> + + + + + + + } + /> + + ); }; diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index 4974c06ca..460f8c468 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -643,7 +643,9 @@ "whitespace_chars": "Whitespace characters", "whitespace_chars_desc": "These characters will be replaced with whitespace in the title" }, - "scene_tools": "Scene Tools" + "scene_tools": "Scene Tools", + "image_tools": "Image Tools", + "image_duplicate_checker": "Image duplicate checker" }, "ui": { "abbreviate_counters": { @@ -1118,6 +1120,7 @@ "distance": "Distance", "donate": "Donate", "dupe_check": { + "missing_phash_warning": "Missing phashes for {count} images. Please run the phash generation task.", "description": "Levels below 'Exact' can take longer to calculate. False positives might also be returned on lower accuracy levels.", "duration_diff": "Maximum Duration Difference", "duration_options": { @@ -1133,12 +1136,12 @@ "medium": "Medium" }, "search_accuracy_label": "Search Accuracy", - "select_all_but_largest_file": "Select every file in each duplicated group, except the largest file", - "select_all_but_largest_resolution": "Select every file in each duplicated group, except the file with highest resolution", + "select_all_but_largest_file": "Keep the largest file (select all but the largest)", + "select_all_but_largest_resolution": "Keep the highest resolution file (select all but the highest resolution)", "select_none": "Select None", - "select_oldest": "Select the oldest file in the duplicate group", + "select_oldest": "Keep the oldest file (select all but the oldest)", "select_options": "Select Options…", - "select_youngest": "Select the youngest file in the duplicate group", + "select_youngest": "Keep the youngest file (select all but the youngest)", "title": "Duplicate Scenes" }, "duplicated": "Duplicated",