mirror of
https://github.com/stashapp/stash.git
synced 2026-04-19 21:41:42 +02:00
acc5438af2
This update provides significant performance improvements for both image and scene duplicate searching: 1. Optimized the core Hamming distance algorithm in pkg/utils/phash.go: - Uses native CPU popcount instructions (math/bits) for bit counting. - Pre-calculates hash values to eliminate object allocations in the hot loop. - Halves the number of comparisons by leveraging the symmetry of the Hamming distance. - The loop is now several orders of magnitude faster and allocation-free. 2. Solved the N+1 database query bottleneck: - Replaced individual database lookups for each duplicate group with a single batched query for all duplicate IDs. - This optimization was applied to both Image and Scene repositories. 3. Simplified the SQL fast path for exact image matches to remove redundant table joins.
92 lines
2.1 KiB
Go
92 lines
2.1 KiB
Go
package utils
|
|
|
|
import (
|
|
"math"
|
|
"math/bits"
|
|
"strconv"
|
|
|
|
"github.com/corona10/goimagehash"
|
|
"github.com/stashapp/stash/pkg/sliceutil"
|
|
)
|
|
|
|
type Phash struct {
|
|
ID int `db:"id"`
|
|
Hash int64 `db:"phash"`
|
|
Duration float64 `db:"duration"`
|
|
Neighbors []int
|
|
Bucket int
|
|
}
|
|
|
|
func FindDuplicates(hashes []*Phash, distance int, durationDiff float64) [][]int {
|
|
// Pre-calculate hash values to avoid allocations and method calls in the inner loop
|
|
uintHashes := make([]uint64, len(hashes))
|
|
for i, h := range hashes {
|
|
uintHashes[i] = uint64(h.Hash)
|
|
}
|
|
|
|
for i, subject := range hashes {
|
|
subjectHash := uintHashes[i]
|
|
for j := i + 1; j < len(hashes); j++ {
|
|
neighbor := hashes[j]
|
|
if subject.ID == neighbor.ID {
|
|
continue
|
|
}
|
|
|
|
// Check duration if applicable (for scenes)
|
|
if durationDiff >= 0 {
|
|
if subject.Duration > 0 && neighbor.Duration > 0 {
|
|
if math.Abs(subject.Duration-neighbor.Duration) > durationDiff {
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
|
|
neighborHash := uintHashes[j]
|
|
// Hamming distance using native bit counting
|
|
if bits.OnesCount64(subjectHash^neighborHash) <= distance {
|
|
subject.Neighbors = append(subject.Neighbors, j)
|
|
neighbor.Neighbors = append(neighbor.Neighbors, i)
|
|
}
|
|
}
|
|
}
|
|
|
|
var buckets [][]int
|
|
for _, subject := range hashes {
|
|
if len(subject.Neighbors) > 0 && subject.Bucket == -1 {
|
|
bucket := len(buckets)
|
|
ids := []int{subject.ID}
|
|
subject.Bucket = bucket
|
|
findNeighbors(bucket, subject.Neighbors, hashes, &ids)
|
|
|
|
if len(ids) > 1 {
|
|
buckets = append(buckets, ids)
|
|
}
|
|
}
|
|
}
|
|
|
|
return buckets
|
|
}
|
|
|
|
func findNeighbors(bucket int, neighbors []int, hashes []*Phash, ids *[]int) {
|
|
for _, id := range neighbors {
|
|
hash := hashes[id]
|
|
if hash.Bucket == -1 {
|
|
hash.Bucket = bucket
|
|
*ids = sliceutil.AppendUnique(*ids, hash.ID)
|
|
findNeighbors(bucket, hash.Neighbors, hashes, ids)
|
|
}
|
|
}
|
|
}
|
|
|
|
func PhashToString(phash int64) string {
|
|
return strconv.FormatUint(uint64(phash), 16)
|
|
}
|
|
|
|
func StringToPhash(s string) (int64, error) {
|
|
ret, err := strconv.ParseUint(s, 16, 64)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return int64(ret), nil
|
|
}
|