Backup_Repos/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-06 16:34:02 +01:00
Code Issues Projects Releases Wiki Activity

Anonymise missing fields (#5244)

Browse source 
* Anonymise missing fields:
- galleries.photographer
- performers.disambiguation
- gallery_urls

* Anonymise captions and saved filters
This commit is contained in:
WithoutPants 2024-09-11 11:50:27 +10:00 committed by GitHub
parent f7a164ffe5
commit 5407596e0d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 95 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
pkg/sqlite/anonymise.go
View file

@ -51,6 +51,7 @@ func (db *Anonymiser) Anonymise(ctx context.Context) error {
func() error { return db.clearWatchHistory() },
func() error { return db.anonymiseFolders(ctx) },
func() error { return db.anonymiseFiles(ctx) },
func() error { return db.anonymiseCaptions(ctx) },
func() error { return db.anonymiseFingerprints(ctx) },
func() error { return db.anonymiseScenes(ctx) },
func() error { return db.anonymiseMarkers(ctx) },
@ -60,6 +61,7 @@ func (db *Anonymiser) Anonymise(ctx context.Context) error {
func() error { return db.anonymiseStudios(ctx) },
func() error { return db.anonymiseTags(ctx) },
func() error { return db.anonymiseGroups(ctx) },
func() error { return db.anonymiseSavedFilters(ctx) },
func() error { return db.Optimise(ctx) },
})
}(); err != nil {
@ -173,6 +175,20 @@ func (db *Anonymiser) anonymiseFiles(ctx context.Context) error {
})
}
func (db *Anonymiser) anonymiseCaptions(ctx context.Context) error {
logger.Infof("Anonymising captions")
return txn.WithTxn(ctx, db, func(ctx context.Context) error {
table := goqu.T(videoCaptionsTable)
stmt := dialect.Update(table).Set(goqu.Record{"filename": goqu.Cast(table.Col("file_id"), "VARCHAR")})
if _, err := exec(ctx, stmt); err != nil {
return fmt.Errorf("anonymising %s: %w", table.GetTable(), err)
}
return nil
})
}
func (db *Anonymiser) anonymiseFingerprints(ctx context.Context) error {
logger.Infof("Anonymising fingerprints")
table := fingerprintTableMgr.table
@ -446,6 +462,7 @@ func (db *Anonymiser) anonymiseGalleries(ctx context.Context) error {
table.Col(idColumn),
table.Col("title"),
table.Col("details"),
table.Col("photographer"),
).Where(table.Col(idColumn).Gt(lastID)).Limit(1000)
gotSome = false
@ -453,15 +470,17 @@ func (db *Anonymiser) anonymiseGalleries(ctx context.Context) error {
const single = false
return queryFunc(ctx, query, single, func(rows *sqlx.Rows) error {
var (
id int
title sql.NullString
details sql.NullString
id int
title sql.NullString
details sql.NullString
photographer sql.NullString
)
if err := rows.Scan(
&id,
&title,
&details,
&photographer,
); err != nil {
return err
}
@ -469,6 +488,7 @@ func (db *Anonymiser) anonymiseGalleries(ctx context.Context) error {
set := goqu.Record{}
db.obfuscateNullString(set, "title", title)
db.obfuscateNullString(set, "details", details)
db.obfuscateNullString(set, "photographer", photographer)
if len(set) > 0 {
stmt := dialect.Update(table).Set(set).Where(table.Col(idColumn).Eq(id))
@ -493,6 +513,10 @@ func (db *Anonymiser) anonymiseGalleries(ctx context.Context) error {
}
}
if err := db.anonymiseURLs(ctx, goqu.T(galleriesURLsTable), "gallery_id"); err != nil {
return err
}
return nil
}
@ -508,6 +532,7 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error {
query := dialect.From(table).Select(
table.Col(idColumn),
table.Col("name"),
table.Col("disambiguation"),
table.Col("details"),
table.Col("tattoos"),
table.Col("piercings"),
@ -518,16 +543,18 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error {
const single = false
return queryFunc(ctx, query, single, func(rows *sqlx.Rows) error {
var (
id int
name sql.NullString
details sql.NullString
tattoos sql.NullString
piercings sql.NullString
id int
name sql.NullString
disambiguation sql.NullString
details sql.NullString
tattoos sql.NullString
piercings sql.NullString
)
if err := rows.Scan(
&id,
&name,
&disambiguation,
&details,
&tattoos,
&piercings,
@ -537,6 +564,7 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error {
set := goqu.Record{}
db.obfuscateNullString(set, "name", name)
db.obfuscateNullString(set, "disambiguation", disambiguation)
db.obfuscateNullString(set, "details", details)
db.obfuscateNullString(set, "tattoos", tattoos)
db.obfuscateNullString(set, "piercings", piercings)
@ -914,6 +942,65 @@ func (db *Anonymiser) anonymiseGroups(ctx context.Context) error {
return nil
}
func (db *Anonymiser) anonymiseSavedFilters(ctx context.Context) error {
logger.Infof("Anonymising saved filters")
table := savedFilterTableMgr.table
lastID := 0
total := 0
const logEvery = 10000
for gotSome := true; gotSome; {
if err := txn.WithTxn(ctx, db, func(ctx context.Context) error {
query := dialect.From(table).Select(
table.Col(idColumn),
table.Col("name"),
).Where(table.Col(idColumn).Gt(lastID)).Limit(1000)
gotSome = false
const single = false
return queryFunc(ctx, query, single, func(rows *sqlx.Rows) error {
var (
id int
name sql.NullString
)
if err := rows.Scan(
&id,
&name,
); err != nil {
return err
}
set := goqu.Record{}
db.obfuscateNullString(set, "name", name)
if len(set) > 0 {
stmt := dialect.Update(table).Set(set).Where(table.Col(idColumn).Eq(id))
if _, err := exec(ctx, stmt); err != nil {
return fmt.Errorf("anonymising %s: %w", table.GetTable(), err)
}
}
lastID = id
gotSome = true
total++
if total%logEvery == 0 {
logger.Infof("Anonymised %d saved filters", total)
}
return nil
})
}); err != nil {
return err
}
}
return nil
}
func (db *Anonymiser) anonymiseText(ctx context.Context, table exp.IdentifierExpression, column string, value string) error {
set := goqu.Record{}
set[column] = db.obfuscateString(value, letters)