This commit is contained in:
notsafeforgit 2026-05-08 05:56:16 +02:00 committed by GitHub
commit a141d16714
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 988 additions and 60 deletions

View file

@ -53,6 +53,9 @@ type Query {
duration_diff: Float
): [[Scene!]!]!
"Find duplicate images"
findDuplicateImages(distance: Int! = 0): [[Image!]!]!
"Return valid stream paths"
sceneStreams(id: ID): [SceneStreamEndpoint!]!

View file

@ -134,3 +134,14 @@ func (r *queryResolver) AllImages(ctx context.Context) (ret []*models.Image, err
return ret, nil
}
func (r *queryResolver) FindDuplicateImages(ctx context.Context, distance int) (ret [][]*models.Image, err error) {
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
ret, err = r.repository.Image.FindDuplicates(ctx, distance)
return err
}); err != nil {
return nil, err
}
return ret, nil
}

View file

@ -370,6 +370,29 @@ func (_m *ImageReaderWriter) FindByZipFileID(ctx context.Context, zipFileID mode
return r0, r1
}
// FindDuplicates provides a mock function with given fields: ctx, distance
func (_m *ImageReaderWriter) FindDuplicates(ctx context.Context, distance int) ([][]*models.Image, error) {
ret := _m.Called(ctx, distance)
var r0 [][]*models.Image
if rf, ok := ret.Get(0).(func(context.Context, int) [][]*models.Image); ok {
r0 = rf(ctx, distance)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([][]*models.Image)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int) error); ok {
r1 = rf(ctx, distance)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FindMany provides a mock function with given fields: ctx, ids
func (_m *ImageReaderWriter) FindMany(ctx context.Context, ids []int) ([]*models.Image, error) {
ret := _m.Called(ctx, ids)

View file

@ -19,6 +19,7 @@ type ImageFinder interface {
FindByZipFileID(ctx context.Context, zipFileID FileID) ([]*Image, error)
FindByGalleryID(ctx context.Context, galleryID int) ([]*Image, error)
FindByGalleryIDIndex(ctx context.Context, galleryID int, index uint) (*Image, error)
FindDuplicates(ctx context.Context, distance int) ([][]*Image, error)
}
// ImageQueryer provides methods to query images.

View file

@ -6,11 +6,13 @@ import (
"errors"
"fmt"
"path/filepath"
"slices"
"strconv"
"strings"
"github.com/jmoiron/sqlx"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/utils"
"gopkg.in/guregu/null.v4"
"gopkg.in/guregu/null.v4/zero"
@ -409,6 +411,11 @@ func (qb *ImageStore) Find(ctx context.Context, id int) (*models.Image, error) {
func (qb *ImageStore) FindMany(ctx context.Context, ids []int) ([]*models.Image, error) {
images := make([]*models.Image, len(ids))
idToIndex := make(map[int]int, len(ids))
for i, id := range ids {
idToIndex[id] = i
}
if err := batchExec(ids, defaultBatchSize, func(batch []int) error {
q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(batch))
unsorted, err := qb.getMany(ctx, q)
@ -417,8 +424,9 @@ func (qb *ImageStore) FindMany(ctx context.Context, ids []int) ([]*models.Image,
}
for _, s := range unsorted {
i := slices.Index(ids, s.ID)
images[i] = s
if i, ok := idToIndex[s.ID]; ok {
images[i] = s
}
}
return nil
@ -837,7 +845,7 @@ func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFi
)
filepathColumn := "folders.path || '" + string(filepath.Separator) + "' || files.basename"
searchColumns := []string{"images.title", "images.details", filepathColumn, "files_fingerprints.fingerprint"}
searchColumns := []string{"images.title", filepathColumn, "files_fingerprints.fingerprint"}
query.parseQueryString(searchColumns, *q)
}
@ -1093,3 +1101,94 @@ func (qb *ImageStore) UpdateTags(ctx context.Context, imageID int, tagIDs []int)
func (qb *ImageStore) GetURLs(ctx context.Context, imageID int) ([]string, error) {
return imagesURLsTableMgr.get(ctx, imageID)
}
var findExactImageDuplicateQuery = `
SELECT GROUP_CONCAT(DISTINCT image_id) as ids
FROM (
SELECT images_files.image_id
, files.size as file_size
, files_fingerprints.fingerprint as phash
FROM images_files
JOIN files ON images_files.file_id = files.id
JOIN files_fingerprints ON images_files.file_id = files_fingerprints.file_id
WHERE files_fingerprints.type = 'phash'
AND files_fingerprints.fingerprint != zeroblob(8)
AND files_fingerprints.fingerprint != ''
)
GROUP BY phash
HAVING COUNT(DISTINCT image_id) > 1
ORDER BY SUM(file_size) DESC;
`
func (qb *ImageStore) FindDuplicates(ctx context.Context, distance int) ([][]*models.Image, error) {
var dupeIds [][]int
if distance == 0 {
var ids []string
if err := dbWrapper.Select(ctx, &ids, findExactImageDuplicateQuery); err != nil {
return nil, err
}
for _, id := range ids {
strIds := strings.Split(id, ",")
var imageIds []int
for _, strId := range strIds {
if intId, err := strconv.Atoi(strId); err == nil {
imageIds = sliceutil.AppendUnique(imageIds, intId)
}
}
// filter out
if len(imageIds) > 1 {
dupeIds = append(dupeIds, imageIds)
}
}
} else {
query := `
SELECT images.id, files_fingerprints.fingerprint as phash
FROM images
JOIN images_files ON images.id = images_files.image_id
JOIN files_fingerprints ON images_files.file_id = files_fingerprints.file_id
WHERE files_fingerprints.type = 'phash'`
var hashes []*utils.Phash
if err := imageRepository.queryFunc(ctx, query, nil, false, func(rows *sqlx.Rows) error {
phash := utils.Phash{
Bucket: -1,
Duration: -1,
}
if err := rows.StructScan(&phash); err != nil {
return err
}
hashes = append(hashes, &phash)
return nil
}); err != nil {
return nil, err
}
dupeIds = utils.FindDuplicates(hashes, distance, -1)
}
var allIds []int
for _, comp := range dupeIds {
allIds = append(allIds, comp...)
}
if len(allIds) == 0 {
return nil, nil
}
allImages, err := qb.FindMany(ctx, allIds)
if err != nil {
return nil, err
}
var result [][]*models.Image
offset := 0
for _, comp := range dupeIds {
group := allImages[offset : offset+len(comp)]
result = append(result, group)
offset += len(comp)
}
return result, nil
}

View file

@ -189,6 +189,10 @@ func (qb *imageFilterHandler) missingCriterionHandler(isMissing *string) criteri
case "tags":
imageRepository.tags.leftJoin(f, "tags_join", "images.id")
f.addWhere("tags_join.image_id IS NULL")
case "phash":
f.addInnerJoin("images_files", "", "images_files.image_id = images.id")
f.addLeftJoin(fingerprintTable, "fingerprints_phash", "images_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'")
f.addWhere("fingerprints_phash.fingerprint IS NULL")
default:
if err := validateIsMissing(*isMissing, []string{
"title", "details", "photographer", "date", "code", "rating",

View file

@ -1596,20 +1596,6 @@ func TestImageQueryQ(t *testing.T) {
})
}
func TestImageQueryQ_Details(t *testing.T) {
withTxn(func(ctx context.Context) error {
const imageIdx = 3
q := getImageStringValue(imageIdx, detailsField)
sqb := db.Image
imageQueryQ(ctx, t, sqb, q, imageIdx)
return nil
})
}
func queryImagesWithCount(ctx context.Context, sqb models.ImageReader, imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) ([]*models.Image, int, error) {
result, err := sqb.Query(ctx, models.ImageQueryOptions{
QueryOptions: models.QueryOptions{

View file

@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
@ -533,9 +532,15 @@ func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene,
return nil, err
}
idToIndex := make(map[int]int, len(ids))
for i, id := range ids {
idToIndex[id] = i
}
for _, s := range unsorted {
i := slices.Index(ids, s.ID)
scenes[i] = s
if i, ok := idToIndex[s.ID]; ok {
scenes[i] = s
}
}
for i := range scenes {
@ -1472,11 +1477,26 @@ func (qb *SceneStore) FindDuplicates(ctx context.Context, distance int, duration
dupeIds = utils.FindDuplicates(hashes, distance, durationDiff)
}
var allIds []int
for _, comp := range dupeIds {
allIds = append(allIds, comp...)
}
if len(allIds) == 0 {
return nil, nil
}
allScenes, err := qb.FindMany(ctx, allIds)
if err != nil {
return nil, err
}
var duplicates [][]*models.Scene
for _, sceneIds := range dupeIds {
if scenes, err := qb.FindMany(ctx, sceneIds); err == nil {
duplicates = append(duplicates, scenes)
}
offset := 0
for _, comp := range dupeIds {
group := allScenes[offset : offset+len(comp)]
duplicates = append(duplicates, group)
offset += len(comp)
}
sortByPath(duplicates)

View file

@ -2,14 +2,16 @@ package utils
import (
"math"
"math/bits"
"runtime"
"strconv"
"sync"
"github.com/corona10/goimagehash"
"github.com/stashapp/stash/pkg/sliceutil"
)
type Phash struct {
SceneID int `db:"id"`
ID int `db:"id"`
Hash int64 `db:"phash"`
Duration float64 `db:"duration"`
Neighbors []int
@ -17,35 +19,65 @@ type Phash struct {
}
func FindDuplicates(hashes []*Phash, distance int, durationDiff float64) [][]int {
for i, scene := range hashes {
sceneHash := goimagehash.NewImageHash(uint64(scene.Hash), goimagehash.PHash)
for j, neighbor := range hashes {
if i != j && scene.SceneID != neighbor.SceneID {
neighbourDurationDistance := 0.
if scene.Duration > 0 && neighbor.Duration > 0 {
neighbourDurationDistance = math.Abs(scene.Duration - neighbor.Duration)
}
if (neighbourDurationDistance <= durationDiff) || (durationDiff < 0) {
neighborHash := goimagehash.NewImageHash(uint64(neighbor.Hash), goimagehash.PHash)
neighborDistance, _ := sceneHash.Distance(neighborHash)
if neighborDistance <= distance {
scene.Neighbors = append(scene.Neighbors, j)
// Pre-calculate hash values to avoid allocations and method calls in the inner loop
uintHashes := make([]uint64, len(hashes))
for i, h := range hashes {
uintHashes[i] = uint64(h.Hash)
}
numHashes := len(hashes)
numWorkers := runtime.GOMAXPROCS(0)
var wg sync.WaitGroup
wg.Add(numWorkers)
// Distribute work among workers
for w := 0; w < numWorkers; w++ {
go func(workerID int) {
defer wg.Done()
for i := workerID; i < numHashes; i += numWorkers {
subject := hashes[i]
subjectHash := uintHashes[i]
for j := 0; j < numHashes; j++ {
if i == j {
continue
}
neighbor := hashes[j]
if subject.ID == neighbor.ID {
continue
}
// Check duration if applicable (for scenes)
if durationDiff >= 0 {
if subject.Duration > 0 && neighbor.Duration > 0 {
if math.Abs(subject.Duration-neighbor.Duration) > durationDiff {
continue
}
}
}
neighborHash := uintHashes[j]
// Hamming distance using native bit counting
if bits.OnesCount64(subjectHash^neighborHash) <= distance {
subject.Neighbors = append(subject.Neighbors, j)
}
}
}
}
}(w)
}
var buckets [][]int
for _, scene := range hashes {
if len(scene.Neighbors) > 0 && scene.Bucket == -1 {
bucket := len(buckets)
scenes := []int{scene.SceneID}
scene.Bucket = bucket
findNeighbors(bucket, scene.Neighbors, hashes, &scenes)
wg.Wait()
if len(scenes) > 1 {
buckets = append(buckets, scenes)
var buckets [][]int
for _, subject := range hashes {
if len(subject.Neighbors) > 0 && subject.Bucket == -1 {
bucket := len(buckets)
ids := []int{subject.ID}
subject.Bucket = bucket
findNeighbors(bucket, subject.Neighbors, hashes, &ids)
if len(ids) > 1 {
buckets = append(buckets, ids)
}
}
}
@ -53,13 +85,13 @@ func FindDuplicates(hashes []*Phash, distance int, durationDiff float64) [][]int
return buckets
}
func findNeighbors(bucket int, neighbors []int, hashes []*Phash, scenes *[]int) {
func findNeighbors(bucket int, neighbors []int, hashes []*Phash, ids *[]int) {
for _, id := range neighbors {
hash := hashes[id]
if hash.Bucket == -1 {
hash.Bucket = bucket
*scenes = sliceutil.AppendUnique(*scenes, hash.SceneID)
findNeighbors(bucket, hash.Neighbors, hashes, scenes)
*ids = sliceutil.AppendUnique(*ids, hash.ID)
findNeighbors(bucket, hash.Neighbors, hashes, ids)
}
}
}

View file

@ -35,3 +35,9 @@ query FindImage($id: ID!, $checksum: String) {
...ImageData
}
}
query FindDuplicateImages($distance: Int!) {
findDuplicateImages(distance: $distance) {
...SlimImageData
}
}

View file

@ -82,6 +82,9 @@ const SceneFilenameParser = lazyComponent(
const SceneDuplicateChecker = lazyComponent(
() => import("./components/SceneDuplicateChecker/SceneDuplicateChecker")
);
const ImageDuplicateChecker = lazyComponent(
() => import("./components/ImageDuplicateChecker/ImageDuplicateChecker")
);
const appleRendering = isPlatformUniquelyRenderedByApple();
@ -269,6 +272,10 @@ export const App: React.FC = () => {
path="/sceneDuplicateChecker"
component={SceneDuplicateChecker}
/>
<Route
path="/imageDuplicateChecker"
component={ImageDuplicateChecker}
/>
<Route path="/setup" component={Setup} />
<Route path="/migrate" component={Migrate} />
<PluginRoutes />

View file

@ -0,0 +1,712 @@
import React, { useMemo, useState } from "react";
import {
Button,
Form,
Table,
Row,
Col,
Card,
Dropdown,
ButtonGroup,
OverlayTrigger,
Tooltip,
} from "react-bootstrap";
import { FormattedMessage, useIntl } from "react-intl";
import { Link, useHistory } from "react-router-dom";
import TextUtils from "src/utils/text";
import { HoverPopover } from "../Shared/HoverPopover";
import { TagLink, GalleryLink } from "../Shared/TagLink";
import { PerformerPopoverButton } from "../Shared/PerformerPopoverButton";
import {
faFileAlt,
faImages,
faTag,
faBox,
faExclamationTriangle,
faPencilAlt,
faTrash,
} from "@fortawesome/free-solid-svg-icons";
import { useFindDuplicateImagesQuery } from "src/core/generated-graphql";
import * as GQL from "src/core/generated-graphql";
import { PatchContainerComponent } from "src/patch";
import { FileSize } from "../Shared/FileSize";
import { Pagination } from "src/components/List/Pagination";
import { DeleteImagesDialog } from "../Images/DeleteImagesDialog";
import { EditImagesDialog } from "../Images/EditImagesDialog";
import { Icon } from "../Shared/Icon";
import { LoadingIndicator } from "../Shared/LoadingIndicator";
import { ErrorMessage } from "../Shared/ErrorMessage";
const CLASSNAME = "duplicate-checker";
const ImageDuplicateCheckerSection = PatchContainerComponent(
"ImageDuplicateCheckerSection"
);
const ImageDuplicateChecker: React.FC = () => {
const intl = useIntl();
const history = useHistory();
const query = new URLSearchParams(history.location.search);
const currentPage = Number.parseInt(query.get("page") ?? "1", 10);
const pageSize = Number.parseInt(query.get("size") ?? "20", 10);
const hashDistance = Number.parseInt(query.get("distance") ?? "0", 10);
const [currentPageSize, setCurrentPageSize] = useState(pageSize);
const [checkedImages, setCheckedImages] = useState<Record<string, boolean>>(
{}
);
const [selectedImages, setSelectedImages] =
useState<GQL.SlimImageDataFragment[]>();
const [deletingImages, setDeletingImages] = useState(false);
const [editingImages, setEditingImages] = useState(false);
const { data: missingPhash } = GQL.useFindImagesQuery({
variables: {
filter: {
per_page: 0,
},
image_filter: {
is_missing: "phash",
},
},
});
function maybeRenderMissingPhashWarning() {
const missingPhashes = missingPhash?.findImages.count ?? 0;
if (missingPhashes > 0) {
return (
<p className="lead">
<Icon icon={faExclamationTriangle} className="text-warning" />
<FormattedMessage
id="dupe_check.missing_phash_warning"
values={{ count: missingPhashes }}
/>
</p>
);
}
}
const { data, loading, refetch } = useFindDuplicateImagesQuery({
variables: { distance: hashDistance },
fetchPolicy: "no-cache",
});
const getGroupTotalSize = (group: GQL.SlimImageDataFragment[]) => {
return group.reduce((groupTotal, img) => {
const imgTotal = img.visual_files.reduce(
(fileTotal, file) => fileTotal + (file.size ?? 0),
0
);
return groupTotal + imgTotal;
}, 0);
};
const allGroups = useMemo(() => {
const groups = data?.findDuplicateImages ?? [];
const groupSizes = new Map<GQL.SlimImageDataFragment[], number>();
groups.forEach((group) => {
groupSizes.set(group, getGroupTotalSize(group));
});
return [...groups].sort((a, b) => {
return (groupSizes.get(b) ?? 0) - (groupSizes.get(a) ?? 0);
});
}, [data?.findDuplicateImages]);
const pagedGroups = useMemo(() => {
const start = (currentPage - 1) * pageSize;
return allGroups.slice(start, start + pageSize);
}, [allGroups, currentPage, pageSize]);
const checkCount = Object.keys(checkedImages).filter(
(id) => checkedImages[id]
).length;
const handleCheck = (checked: boolean, imageID: string) => {
setCheckedImages({ ...checkedImages, [imageID]: checked });
};
const handleDeleteChecked = () => {
setSelectedImages(allGroups.flat().filter((i) => checkedImages[i.id]));
setDeletingImages(true);
};
const onEdit = () => {
setSelectedImages(allGroups.flat().filter((i) => checkedImages[i.id]));
setEditingImages(true);
setCheckedImages({});
};
const onDeleteDialogClosed = async (confirmed: boolean) => {
setDeletingImages(false);
setSelectedImages(undefined);
if (confirmed) {
setCheckedImages({});
await refetch();
}
};
const onEditDialogClosed = async (applied: boolean) => {
setEditingImages(false);
setSelectedImages(undefined);
if (applied) {
await refetch();
}
};
const pageOptions = useMemo(() => {
const pageSizes = [
10, 20, 30, 40, 50, 100, 150, 200, 250, 500, 750, 1000, 1250, 1500,
];
const filteredSizes = pageSizes.filter((s, i) => {
return (
allGroups.length > s || i == 0 || allGroups.length > pageSizes[i - 1]
);
});
return filteredSizes.map((size) => {
return (
<option key={size} value={size}>
{size}
</option>
);
});
}, [allGroups.length]);
if (loading) return <LoadingIndicator />;
if (!data) return <ErrorMessage error="Error searching for duplicates." />;
const setQuery = (q: Record<string, string | number | undefined>) => {
const newQuery = new URLSearchParams(query);
for (const key of Object.keys(q)) {
const value = q[key];
if (value !== undefined) {
newQuery.set(key, String(value));
} else {
newQuery.delete(key);
}
}
history.push({ search: newQuery.toString() });
};
const resetCheckboxSelection = () => {
const updatedImages: Record<string, boolean> = {};
Object.keys(checkedImages).forEach((imageKey) => {
updatedImages[imageKey] = false;
});
setCheckedImages(updatedImages);
};
const findLargestImage = (group: GQL.SlimImageDataFragment[]) => {
const totalSize = (image: GQL.SlimImageDataFragment) => {
return image.visual_files.reduce(
(prev: number, f) => Math.max(prev, f.size ?? 0),
0
);
};
return group.reduce((largest, image) => {
const largestSize = totalSize(largest);
const currentSize = totalSize(image);
return currentSize > largestSize ? image : largest;
});
};
const findLargestResolutionImage = (group: GQL.SlimImageDataFragment[]) => {
const imgResolution = (image: GQL.SlimImageDataFragment) => {
return image.visual_files.reduce(
(prev: number, f) => Math.max(prev, (f.height ?? 0) * (f.width ?? 0)),
0
);
};
return group.reduce((largest, image) => {
const largestSize = imgResolution(largest);
const currentSize = imgResolution(image);
return currentSize > largestSize ? image : largest;
});
};
const findFirstFileByAge = (
oldest: boolean,
compareImages: GQL.SlimImageDataFragment[]
) => {
let selectedFile: GQL.ImageFileDataFragment | GQL.VideoFileDataFragment;
let oldestTimestamp: Date | undefined = undefined;
for (const file of compareImages.flatMap((s) => s.visual_files)) {
const timestamp: Date = new Date(file.mod_time);
if (oldest) {
if (oldestTimestamp === undefined || timestamp < oldestTimestamp) {
oldestTimestamp = timestamp;
selectedFile = file;
}
} else {
if (oldestTimestamp === undefined || timestamp > oldestTimestamp) {
oldestTimestamp = timestamp;
selectedFile = file;
}
}
}
return compareImages.find((s) =>
s.visual_files.some((f) => f.id === selectedFile?.id)
);
};
function checkSameResolution(dataGroup: GQL.SlimImageDataFragment[]) {
const resolutions = dataGroup.map((s) => {
return s.visual_files.reduce(
(prev, f) => Math.max(prev, (f.height ?? 0) * (f.width ?? 0)),
0
);
});
return new Set(resolutions).size === 1;
}
const onSelectLargestClick = () => {
setSelectedImages([]);
const checkedArray: Record<string, boolean> = {};
pagedGroups.forEach((group) => {
const largest = findLargestImage(group);
group.forEach((image) => {
if (image !== largest) {
checkedArray[image.id] = true;
}
});
});
setCheckedImages(checkedArray);
};
const onSelectLargestResolutionClick = () => {
setSelectedImages([]);
const checkedArray: Record<string, boolean> = {};
pagedGroups.forEach((group) => {
if (checkSameResolution(group)) return;
const highest = findLargestResolutionImage(group);
group.forEach((image) => {
if (image !== highest) {
checkedArray[image.id] = true;
}
});
});
setCheckedImages(checkedArray);
};
const onSelectByAge = (oldest: boolean) => {
setSelectedImages([]);
const checkedArray: Record<string, boolean> = {};
pagedGroups.forEach((group) => {
const oldestScene = findFirstFileByAge(oldest, group);
group.forEach((image) => {
if (image !== oldestScene) {
checkedArray[image.id] = true;
}
});
});
setCheckedImages(checkedArray);
};
const handleDeleteImage = (image: GQL.SlimImageDataFragment) => {
setSelectedImages([image]);
setDeletingImages(true);
};
function renderPagination() {
return (
<div className="d-flex mt-2 mb-2">
<h6 className="mr-auto align-self-center">
<FormattedMessage
id="dupe_check.found_sets"
values={{ setCount: allGroups.length }}
/>
</h6>
{checkCount > 0 && (
<ButtonGroup>
<OverlayTrigger
overlay={
<Tooltip id="edit">
{intl.formatMessage({ id: "actions.edit" })}
</Tooltip>
}
>
<Button variant="secondary" onClick={onEdit}>
<Icon icon={faPencilAlt} />
</Button>
</OverlayTrigger>
<OverlayTrigger
overlay={
<Tooltip id="delete">
{intl.formatMessage({ id: "actions.delete" })}
</Tooltip>
}
>
<Button variant="danger" onClick={handleDeleteChecked}>
<Icon icon={faTrash} />
</Button>
</OverlayTrigger>
</ButtonGroup>
)}
<Pagination
itemsPerPage={pageSize}
currentPage={currentPage}
totalItems={allGroups.length}
metadataByline={[]}
onChangePage={(newPage) => {
setQuery({ page: newPage === 1 ? undefined : newPage });
resetCheckboxSelection();
}}
/>
<Form.Control
as="select"
className="w-auto ml-2 btn-secondary"
defaultValue={pageSize}
value={currentPageSize}
onChange={(e) => {
setCurrentPageSize(parseInt(e.currentTarget.value, 10));
setQuery({
size:
e.currentTarget.value === "20"
? undefined
: e.currentTarget.value,
});
resetCheckboxSelection();
}}
>
{pageOptions}
</Form.Control>
</div>
);
}
function maybeRenderPopoverButtonGroup(image: GQL.SlimImageDataFragment) {
if (
image.tags.length > 0 ||
image.performers.length > 0 ||
image.galleries.length > 0 ||
image.visual_files.length > 1 ||
image.organized
) {
return (
<ButtonGroup className="flex-wrap">
{image.tags.length > 0 && (
<HoverPopover
placement="bottom"
content={image.tags.map((tag) => (
<TagLink key={tag.id} tag={tag} />
))}
>
<Button className="minimal">
<Icon icon={faTag} />
<span>{image.tags.length}</span>
</Button>
</HoverPopover>
)}
{image.performers.length > 0 && (
<PerformerPopoverButton performers={image.performers} />
)}
{image.galleries.length > 0 && (
<HoverPopover
placement="bottom"
content={image.galleries.map((g) => (
<GalleryLink key={g.id} gallery={g} />
))}
>
<Button className="minimal">
<Icon icon={faImages} />
<span>{image.galleries.length}</span>
</Button>
</HoverPopover>
)}
{image.visual_files.length > 1 && (
<HoverPopover
placement="bottom"
content={
<FormattedMessage
id="files_amount"
values={{
value: intl.formatNumber(image.visual_files.length),
}}
/>
}
>
<Button className="minimal">
<Icon icon={faFileAlt} />
<span>{image.visual_files.length}</span>
</Button>
</HoverPopover>
)}
{image.organized && (
<div>
<Button className="minimal">
<Icon icon={faBox} />
</Button>
</div>
)}
</ButtonGroup>
);
}
}
return (
<Card id="image-duplicate-checker" className="col col-xl-12 mx-auto">
<div className={CLASSNAME}>
<ImageDuplicateCheckerSection>
{deletingImages && selectedImages && (
<DeleteImagesDialog
selected={selectedImages}
onClose={onDeleteDialogClosed}
/>
)}
{editingImages && selectedImages && (
<EditImagesDialog
selected={selectedImages}
onClose={onEditDialogClosed}
/>
)}
<h4>
<FormattedMessage id="config.tools.image_duplicate_checker" />
</h4>
<Form>
<Form.Group>
<Row noGutters>
<Form.Label>
<FormattedMessage id="dupe_check.search_accuracy_label" />
</Form.Label>
<Col xs="auto">
<Form.Control
as="select"
onChange={(e) =>
setQuery({
distance:
e.currentTarget.value === "0"
? undefined
: e.currentTarget.value,
page: undefined,
})
}
defaultValue={hashDistance}
className="input-control ml-4"
>
<option value={0}>
{intl.formatMessage({ id: "dupe_check.options.exact" })}
</option>
<option value={4}>
{intl.formatMessage({ id: "dupe_check.options.high" })}
</option>
<option value={8}>
{intl.formatMessage({ id: "dupe_check.options.medium" })}
</option>
<option value={10}>
{intl.formatMessage({ id: "dupe_check.options.low" })}
</option>
</Form.Control>
</Col>
</Row>
<Form.Text>
<FormattedMessage id="dupe_check.description" />
</Form.Text>
</Form.Group>
<Form.Group>
<Row noGutters>
<Col xs="12">
<Dropdown className="">
<Dropdown.Toggle variant="secondary">
<FormattedMessage id="dupe_check.select_options" />
</Dropdown.Toggle>
<Dropdown.Menu className="bg-secondary text-white">
<Dropdown.Item onClick={() => resetCheckboxSelection()}>
{intl.formatMessage({ id: "dupe_check.select_none" })}
</Dropdown.Item>
<Dropdown.Item
onClick={() => onSelectLargestResolutionClick()}
>
{intl.formatMessage({
id: "dupe_check.select_all_but_largest_resolution",
})}
</Dropdown.Item>
<Dropdown.Item onClick={() => onSelectLargestClick()}>
{intl.formatMessage({
id: "dupe_check.select_all_but_largest_file",
})}
</Dropdown.Item>
<Dropdown.Item onClick={() => onSelectByAge(true)}>
{intl.formatMessage({
id: "dupe_check.select_oldest",
})}
</Dropdown.Item>
<Dropdown.Item onClick={() => onSelectByAge(false)}>
{intl.formatMessage({
id: "dupe_check.select_youngest",
})}
</Dropdown.Item>
</Dropdown.Menu>
</Dropdown>
</Col>
</Row>
</Form.Group>
</Form>
{maybeRenderMissingPhashWarning()}
{renderPagination()}
<Table responsive striped className={`${CLASSNAME}-table`}>
<colgroup>
<col className={`${CLASSNAME}-checkbox`} />
<col className={`${CLASSNAME}-sprite`} />
<col className={`${CLASSNAME}-title`} />
<col className={`${CLASSNAME}-details`} />
<col className={`${CLASSNAME}-filesize`} />
<col className={`${CLASSNAME}-resolution`} />
<col className={`${CLASSNAME}-operations`} />
</colgroup>
<thead>
<tr>
<th> </th>
<th> </th>
<th>{intl.formatMessage({ id: "details" })}</th>
<th> </th>
<th>{intl.formatMessage({ id: "filesize" })}</th>
<th>{intl.formatMessage({ id: "resolution" })}</th>
<th>{intl.formatMessage({ id: "actions.delete" })}</th>
</tr>
</thead>
<tbody>
{pagedGroups.map((group, groupIndex) =>
group.map((image, i) => {
const file = image.visual_files[0];
return (
<React.Fragment key={image.id}>
{i === 0 && groupIndex !== 0 ? (
<tr className="separator" />
) : undefined}
<tr
className={i === 0 ? "duplicate-group" : ""}
key={image.id}
>
<td>
<Form.Check
checked={checkedImages[image.id] || false}
onChange={(e) =>
handleCheck(e.currentTarget.checked, image.id)
}
/>
</td>
<td>
<HoverPopover
content={
<img
src={image.paths.thumbnail || ""}
alt=""
style={{
maxWidth: 600,
maxHeight: 600,
objectFit: "contain",
}}
/>
}
placement="right"
>
<img
src={image.paths.thumbnail || ""}
alt=""
style={{
maxWidth: "120px",
maxHeight: "120px",
objectFit: "contain",
border: checkedImages[image.id]
? "2px solid red"
: "",
}}
/>
</HoverPopover>
</td>
<td className="text-left">
<p>
<Link
to={`/images/${image.id}`}
style={{
fontWeight: checkedImages[image.id]
? "bold"
: "inherit",
textDecoration: checkedImages[image.id]
? "line-through 3px"
: "inherit",
textDecorationColor: checkedImages[image.id]
? "red"
: "inherit",
}}
>
{image.title ||
TextUtils.fileNameFromPath(file?.path ?? "")}
</Link>
</p>
<p className="scene-path">{file?.path ?? ""}</p>
</td>
<td className="scene-details">
{maybeRenderPopoverButtonGroup(image)}
</td>
<td>
<FileSize size={file?.size ?? 0} />
</td>
<td>
{file?.__typename === "ImageFile" ||
file?.__typename === "VideoFile" ? (
<>
{file.width ?? 0}x{file.height ?? 0}
</>
) : (
"N/A"
)}
</td>
<td>
<Button
className="edit-button"
variant="danger"
onClick={() => handleDeleteImage(image)}
>
<FormattedMessage id="actions.delete" />
</Button>
</td>
</tr>
</React.Fragment>
);
})
)}
</tbody>
</Table>
{allGroups.length === 0 && !loading && (
<h4 className="text-center mt-4">No duplicates found.</h4>
)}
{loading && (
<div className="text-center mt-4">
<Icon icon={faBox} spin className="fa-3x" />
<h4 className="mt-2">Loading...</h4>
</div>
)}
{renderPagination()}
</ImageDuplicateCheckerSection>
</div>
</Card>
);
};
export default ImageDuplicateChecker;

View file

@ -61,11 +61,12 @@ export const DeleteImagesDialog: React.FC<IDeleteImageDialogProps> = (
try {
await deleteImage();
Toast.success(toastMessage);
props.onClose(true);
} catch (e) {
Toast.error(e);
props.onClose(false);
}
setIsDeleting(false);
props.onClose(true);
}
function maybeRenderDeleteFileAlert() {

View file

@ -92,9 +92,15 @@ export const SceneDuplicateChecker: React.FC = () => {
const scenes = useMemo(() => {
const groups = data?.findDuplicateScenes ?? [];
const groupSizes = new Map<GQL.SlimSceneDataFragment[], number>();
groups.forEach((group) => {
groupSizes.set(group, getGroupTotalSize(group));
});
// Sort by total file size descending (largest groups first)
return [...groups].sort((a, b) => {
return getGroupTotalSize(b) - getGroupTotalSize(a);
return (groupSizes.get(b) ?? 0) - (groupSizes.get(a) ?? 0);
});
}, [data?.findDuplicateScenes]);

View file

@ -48,6 +48,20 @@ export const SettingsToolsPanel: React.FC = () => {
/>
</SettingsToolsSection>
</SettingSection>
<SettingSection headingID="config.tools.image_tools">
<SettingsToolsSection>
<Setting
heading={
<Link to="/imageDuplicateChecker">
<Button>
<FormattedMessage id="config.tools.image_duplicate_checker" />
</Button>
</Link>
}
/>
</SettingsToolsSection>
</SettingSection>
</>
);
};

View file

@ -643,7 +643,9 @@
"whitespace_chars": "Whitespace characters",
"whitespace_chars_desc": "These characters will be replaced with whitespace in the title"
},
"scene_tools": "Scene Tools"
"scene_tools": "Scene Tools",
"image_tools": "Image Tools",
"image_duplicate_checker": "Image duplicate checker"
},
"ui": {
"abbreviate_counters": {
@ -1118,6 +1120,7 @@
"distance": "Distance",
"donate": "Donate",
"dupe_check": {
"missing_phash_warning": "Missing phashes for {count} images. Please run the phash generation task.",
"description": "Levels below 'Exact' can take longer to calculate. False positives might also be returned on lower accuracy levels.",
"duration_diff": "Maximum Duration Difference",
"duration_options": {
@ -1133,12 +1136,12 @@
"medium": "Medium"
},
"search_accuracy_label": "Search Accuracy",
"select_all_but_largest_file": "Select every file in each duplicated group, except the largest file",
"select_all_but_largest_resolution": "Select every file in each duplicated group, except the file with highest resolution",
"select_all_but_largest_file": "Keep the largest file (select all but the largest)",
"select_all_but_largest_resolution": "Keep the highest resolution file (select all but the highest resolution)",
"select_none": "Select None",
"select_oldest": "Select the oldest file in the duplicate group",
"select_oldest": "Keep the oldest file (select all but the oldest)",
"select_options": "Select Options…",
"select_youngest": "Select the youngest file in the duplicate group",
"select_youngest": "Keep the youngest file (select all but the youngest)",
"title": "Duplicate Scenes"
},
"duplicated": "Duplicated",