Backup_Repos/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-06 08:26:00 +01:00
Code Issues Projects Releases Wiki Activity

Handle modified files where the case of the filename changed on case-insensitive filesystems (#6327)

Browse source 
* Find existing files with case insensitivity if filesystem is case insensitive
* Handle case change in folders
* Optimise to only test file system case sensitivity if the first query found nothing

This limits the overhead to new paths, and adds an extra query for new paths to windows installs
This commit is contained in:
WithoutPants 2025-12-02 12:53:37 +11:00 committed by GitHub
parent 49fd47562e
commit 4017c42fe2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 110 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
internal/api/resolver_query_find_file.go
View file

@ -29,7 +29,7 @@ func (r *queryResolver) FindFile(ctx context.Context, id *string, path *string)
ret = files[0]
}
case path != nil:
ret, err = qb.FindByPath(ctx, *path)
ret, err = qb.FindByPath(ctx, *path, true)
if err == nil && ret == nil {
return errors.New("file not found")
}

2
internal/api/resolver_query_find_folder.go
View file

@ -25,7 +25,7 @@ func (r *queryResolver) FindFolder(ctx context.Context, id *string, path *string
return err
}
case path != nil:
ret, err = qb.FindByPath(ctx, *path)
ret, err = qb.FindByPath(ctx, *path, true)
if err == nil && ret == nil {
return errors.New("folder not found")
}

2
internal/autotag/integration_test.go
View file

@ -225,7 +225,7 @@ func createSceneFile(ctx context.Context, name string, folderStore models.Folder
}
func getOrCreateFolder(ctx context.Context, folderStore models.FolderFinderCreator, folderPath string) (*models.Folder, error) {
f, err := folderStore.FindByPath(ctx, folderPath)
f, err := folderStore.FindByPath(ctx, folderPath, true)
if err != nil {
return nil, fmt.Errorf("getting folder by path: %w", err)
}

4
pkg/file/folder.go
View file

@ -15,7 +15,9 @@ import (
// Does not create any folders in the file system
func GetOrCreateFolderHierarchy(ctx context.Context, fc models.FolderFinderCreator, path string) (*models.Folder, error) {
// get or create folder hierarchy
folder, err := fc.FindByPath(ctx, path)
// assume case sensitive when searching for the folder
const caseSensitive = true
folder, err := fc.FindByPath(ctx, path, caseSensitive)
if err != nil {
return nil, err
}

6
pkg/file/import.go
View file

@ -120,7 +120,7 @@ func (i *Importer) baseFileJSONToBaseFile(ctx context.Context, baseJSON *jsonsch
func (i *Importer) populateZipFileID(ctx context.Context, f *models.DirEntry) error {
zipFilePath := i.Input.DirEntry().ZipFile
if zipFilePath != "" {
zf, err := i.ReaderWriter.FindByPath(ctx, zipFilePath)
zf, err := i.ReaderWriter.FindByPath(ctx, zipFilePath, true)
if err != nil {
return fmt.Errorf("error finding file by path %q: %v", zipFilePath, err)
}
@ -146,7 +146,7 @@ func (i *Importer) Name() string {
func (i *Importer) FindExistingID(ctx context.Context) (*int, error) {
path := i.Input.DirEntry().Path
existing, err := i.ReaderWriter.FindByPath(ctx, path)
existing, err := i.ReaderWriter.FindByPath(ctx, path, true)
if err != nil {
return nil, err
}
@ -176,7 +176,7 @@ func (i *Importer) createFolderHierarchy(ctx context.Context, p string) (*models
}
func (i *Importer) getOrCreateFolder(ctx context.Context, path string, parent *models.Folder) (*models.Folder, error) {
folder, err := i.FolderStore.FindByPath(ctx, path)
folder, err := i.FolderStore.FindByPath(ctx, path, true)
if err != nil {
return nil, err
}

59
pkg/file/scan.go
View file

@ -443,7 +443,10 @@ func (s *scanJob) getFolderID(ctx context.Context, path string) (*models.FolderI
return &v, nil
}
ret, err := s.Repository.Folder.FindByPath(ctx, path)
// assume case sensitive when searching for the folder
const caseSensitive = true
ret, err := s.Repository.Folder.FindByPath(ctx, path, caseSensitive)
if err != nil {
return nil, err
}
@ -473,7 +476,10 @@ func (s *scanJob) getZipFileID(ctx context.Context, zipFile *scanFile) (*models.
return &v, nil
}
ret, err := s.Repository.File.FindByPath(ctx, path)
// assume case sensitive when searching for the zip file
const caseSensitive = true
ret, err := s.Repository.File.FindByPath(ctx, path, caseSensitive)
if err != nil {
return nil, fmt.Errorf("getting zip file ID for %q: %w", path, err)
}
@ -493,11 +499,26 @@ func (s *scanJob) handleFolder(ctx context.Context, file scanFile) error {
defer s.incrementProgress(file)
// determine if folder already exists in data store (by path)
f, err := s.Repository.Folder.FindByPath(ctx, path)
// assume case sensitive by default
f, err := s.Repository.Folder.FindByPath(ctx, path, true)
if err != nil {
return fmt.Errorf("checking for existing folder %q: %w", path, err)
}
// #1426 / #6326 - if folder is in a case-insensitive filesystem, then try
// case insensitive searching
// assume case sensitive if in zip
if f == nil && file.ZipFileID == nil {
caseSensitive, _ := file.fs.IsPathCaseSensitive(file.Path)
if !caseSensitive {
f, err = s.Repository.Folder.FindByPath(ctx, path, false)
if err != nil {
return fmt.Errorf("checking for existing folder %q: %w", path, err)
}
}
}
// if folder not exists, create it
if f == nil {
f, err = s.onNewFolder(ctx, file)
@ -611,10 +632,18 @@ func (s *scanJob) onExistingFolder(ctx context.Context, f scanFile, existing *mo
// update if mod time is changed
entryModTime := f.ModTime
if !entryModTime.Equal(existing.ModTime) {
existing.Path = f.Path
existing.ModTime = entryModTime
update = true
}
// #6326 - update if path has changed - should only happen if case is
// changed and filesystem is case insensitive
if existing.Path != f.Path {
existing.Path = f.Path
update = true
}
// update if zip file ID has changed
fZfID := f.ZipFileID
existingZfID := existing.ZipFileID
@ -647,15 +676,31 @@ func (s *scanJob) handleFile(ctx context.Context, f scanFile) error {
defer s.incrementProgress(f)
var ff models.File
// don't use a transaction to check if new or existing
if err := s.withDB(ctx, func(ctx context.Context) error {
// determine if file already exists in data store
// assume case sensitive when searching for the file to begin with
var err error
ff, err = s.Repository.File.FindByPath(ctx, f.Path)
ff, err = s.Repository.File.FindByPath(ctx, f.Path, true)
if err != nil {
return fmt.Errorf("checking for existing file %q: %w", f.Path, err)
}
// #1426 / #6326 - if file is in a case-insensitive filesystem, then try
// case insensitive search
// assume case sensitive if in zip
if ff == nil && f.ZipFileID != nil {
caseSensitive, _ := f.fs.IsPathCaseSensitive(f.Path)
if !caseSensitive {
ff, err = s.Repository.File.FindByPath(ctx, f.Path, false)
if err != nil {
return fmt.Errorf("checking for existing file %q: %w", f.Path, err)
}
}
}
if ff == nil {
// returns a file only if it is actually new
ff, err = s.onNewFile(ctx, f)
@ -879,6 +924,7 @@ func (s *scanJob) handleRename(ctx context.Context, f models.File, fp []models.F
// #1426 - if file exists but is a case-insensitive match for the
// original filename, and the filesystem is case-insensitive
// then treat it as a move
// #6326 - this should now be handled earlier, and this shouldn't be necessary
if caseSensitive, _ := fs.IsPathCaseSensitive(other.Base().Path); !caseSensitive {
// treat as a move
missing = append(missing, other)
@ -1026,7 +1072,8 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing model
path := base.Path
fileModTime := f.ModTime
updated := !fileModTime.Equal(base.ModTime)
// #6326 - also force a rescan if the basename changed
updated := !fileModTime.Equal(base.ModTime) || base.Basename != f.Basename
forceRescan := s.options.Rescan
if !updated && !forceRescan {
@ -1041,6 +1088,8 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing model
logger.Infof("%s has been updated: rescanning", path)
}
// #6326 - update basename in case it changed
base.Basename = f.Basename
base.ModTime = fileModTime
base.Size = f.Size
base.UpdatedAt = time.Now()

2
pkg/file/video/caption.go
View file

@ -97,7 +97,7 @@ func AssociateCaptions(ctx context.Context, captionPath string, txnMgr txn.Manag
captionPrefix := getCaptionPrefix(captionPath)
if err := txn.WithTxn(ctx, txnMgr, func(ctx context.Context) error {
var err error
files, er := fqb.FindAllByPath(ctx, captionPrefix+"*")
files, er := fqb.FindAllByPath(ctx, captionPrefix+"*", true)
if er != nil {
return fmt.Errorf("searching for scene %s: %w", captionPrefix, er)

4
pkg/gallery/import.go
View file

@ -265,7 +265,7 @@ func (i *Importer) populateFilesFolder(ctx context.Context) error {
for _, ref := range i.Input.ZipFiles {
path := ref
f, err := i.FileFinder.FindByPath(ctx, path)
f, err := i.FileFinder.FindByPath(ctx, path, true)
if err != nil {
return fmt.Errorf("error finding file: %w", err)
}
@ -281,7 +281,7 @@ func (i *Importer) populateFilesFolder(ctx context.Context) error {
if i.Input.FolderPath != "" {
path := i.Input.FolderPath
f, err := i.FolderFinder.FindByPath(ctx, path)
f, err := i.FolderFinder.FindByPath(ctx, path, true)
if err != nil {
return fmt.Errorf("error finding folder: %w", err)
}

2
pkg/image/import.go
View file

@ -110,7 +110,7 @@ func (i *Importer) populateFiles(ctx context.Context) error {
for _, ref := range i.Input.Files {
path := ref
f, err := i.FileFinder.FindByPath(ctx, path)
f, err := i.FileFinder.FindByPath(ctx, path, true)
if err != nil {
return fmt.Errorf("error finding file: %w", err)
}

28
pkg/models/mocks/FileReaderWriter.go
View file

@ -130,13 +130,13 @@ func (_m *FileReaderWriter) Find(ctx context.Context, id ...models.FileID) ([]mo
return r0, r1
}
// FindAllByPath provides a mock function with given fields: ctx, path
func (_m *FileReaderWriter) FindAllByPath(ctx context.Context, path string) ([]models.File, error) {
ret := _m.Called(ctx, path)
// FindAllByPath provides a mock function with given fields: ctx, path, caseSensitive
func (_m *FileReaderWriter) FindAllByPath(ctx context.Context, path string, caseSensitive bool) ([]models.File, error) {
ret := _m.Called(ctx, path, caseSensitive)
var r0 []models.File
if rf, ok := ret.Get(0).(func(context.Context, string) []models.File); ok {
r0 = rf(ctx, path)
if rf, ok := ret.Get(0).(func(context.Context, string, bool) []models.File); ok {
r0 = rf(ctx, path, caseSensitive)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]models.File)
@ -144,8 +144,8 @@ func (_m *FileReaderWriter) FindAllByPath(ctx context.Context, path string) ([]m
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, path)
if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok {
r1 = rf(ctx, path, caseSensitive)
} else {
r1 = ret.Error(1)
}
@ -222,13 +222,13 @@ func (_m *FileReaderWriter) FindByFingerprint(ctx context.Context, fp models.Fin
return r0, r1
}
// FindByPath provides a mock function with given fields: ctx, path
func (_m *FileReaderWriter) FindByPath(ctx context.Context, path string) (models.File, error) {
ret := _m.Called(ctx, path)
// FindByPath provides a mock function with given fields: ctx, path, caseSensitive
func (_m *FileReaderWriter) FindByPath(ctx context.Context, path string, caseSensitive bool) (models.File, error) {
ret := _m.Called(ctx, path, caseSensitive)
var r0 models.File
if rf, ok := ret.Get(0).(func(context.Context, string) models.File); ok {
r0 = rf(ctx, path)
if rf, ok := ret.Get(0).(func(context.Context, string, bool) models.File); ok {
r0 = rf(ctx, path, caseSensitive)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(models.File)
@ -236,8 +236,8 @@ func (_m *FileReaderWriter) FindByPath(ctx context.Context, path string) (models
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, path)
if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok {
r1 = rf(ctx, path, caseSensitive)
} else {
r1 = ret.Error(1)
}

14
pkg/models/mocks/FolderReaderWriter.go
View file

@ -132,13 +132,13 @@ func (_m *FolderReaderWriter) FindByParentFolderID(ctx context.Context, parentFo
return r0, r1
}
// FindByPath provides a mock function with given fields: ctx, path
func (_m *FolderReaderWriter) FindByPath(ctx context.Context, path string) (*models.Folder, error) {
ret := _m.Called(ctx, path)
// FindByPath provides a mock function with given fields: ctx, path, caseSensitive
func (_m *FolderReaderWriter) FindByPath(ctx context.Context, path string, caseSensitive bool) (*models.Folder, error) {
ret := _m.Called(ctx, path, caseSensitive)
var r0 *models.Folder
if rf, ok := ret.Get(0).(func(context.Context, string) *models.Folder); ok {
r0 = rf(ctx, path)
if rf, ok := ret.Get(0).(func(context.Context, string, bool) *models.Folder); ok {
r0 = rf(ctx, path, caseSensitive)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*models.Folder)
@ -146,8 +146,8 @@ func (_m *FolderReaderWriter) FindByPath(ctx context.Context, path string) (*mod
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, path)
if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok {
r1 = rf(ctx, path, caseSensitive)
} else {
r1 = ret.Error(1)
}

4
pkg/models/repository_file.go
View file

@ -13,9 +13,9 @@ type FileGetter interface {
// FileFinder provides methods to find files.
type FileFinder interface {
FileGetter
FindAllByPath(ctx context.Context, path string) ([]File, error)
FindAllByPath(ctx context.Context, path string, caseSensitive bool) ([]File, error)
FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]File, error)
FindByPath(ctx context.Context, path string) (File, error)
FindByPath(ctx context.Context, path string, caseSensitive bool) (File, error)
FindByFingerprint(ctx context.Context, fp Fingerprint) ([]File, error)
FindByZipFileID(ctx context.Context, zipFileID FileID) ([]File, error)
FindByFileInfo(ctx context.Context, info fs.FileInfo, size int64) ([]File, error)

2
pkg/models/repository_folder.go
View file

@ -12,7 +12,7 @@ type FolderGetter interface {
type FolderFinder interface {
FolderGetter
FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]*Folder, error)
FindByPath(ctx context.Context, path string) (*Folder, error)
FindByPath(ctx context.Context, path string, caseSensitive bool) (*Folder, error)
FindByZipFileID(ctx context.Context, zipFileID FileID) ([]*Folder, error)
FindByParentFolderID(ctx context.Context, parentFolderID FolderID) ([]*Folder, error)
}

2
pkg/scene/import.go
View file

@ -164,7 +164,7 @@ func (i *Importer) populateFiles(ctx context.Context) error {
for _, ref := range i.Input.Files {
path := ref
f, err := i.FileFinder.FindByPath(ctx, path)
f, err := i.FileFinder.FindByPath(ctx, path, true)
if err != nil {
return fmt.Errorf("error finding file: %w", err)
}

8
pkg/sqlite/file.go
View file

@ -625,9 +625,9 @@ func (qb *FileStore) find(ctx context.Context, id models.FileID) (models.File, e
}
// FindByPath returns the first file that matches the given path. Wildcard characters are supported.
func (qb *FileStore) FindByPath(ctx context.Context, p string) (models.File, error) {
func (qb *FileStore) FindByPath(ctx context.Context, p string, caseSensitive bool) (models.File, error) {
ret, err := qb.FindAllByPath(ctx, p)
ret, err := qb.FindAllByPath(ctx, p, caseSensitive)
if err != nil {
return nil, err
@ -642,7 +642,7 @@ func (qb *FileStore) FindByPath(ctx context.Context, p string) (models.File, err
// FindAllByPath returns all the files that match the given path.
// Wildcard characters are supported.
func (qb *FileStore) FindAllByPath(ctx context.Context, p string) ([]models.File, error) {
func (qb *FileStore) FindAllByPath(ctx context.Context, p string, caseSensitive bool) ([]models.File, error) {
// separate basename from path
basename := filepath.Base(p)
dirName := filepath.Dir(p)
@ -657,7 +657,7 @@ func (qb *FileStore) FindAllByPath(ctx context.Context, p string) ([]models.File
// like uses case-insensitive matching. Only use like if wildcards are used
q := qb.selectDataset().Prepared(true)
if strings.Contains(basename, "%") || strings.Contains(dirName, "%") {
if strings.Contains(basename, "%") || strings.Contains(dirName, "%") || !caseSensitive {
q = q.Where(
folderTable.Col("path").Like(dirName),
table.Col("basename").Like(basename),

2
pkg/sqlite/file_test.go
View file

@ -551,7 +551,7 @@ func Test_FileStore_FindByPath(t *testing.T) {
for _, tt := range tests {
runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
assert := assert.New(t)
got, err := qb.FindByPath(ctx, tt.path)
got, err := qb.FindByPath(ctx, tt.path, true)
if (err != nil) != tt.wantErr {
t.Errorf("FileStore.FindByPath() error = %v, wantErr %v", err, tt.wantErr)
return

12
pkg/sqlite/folder.go
View file

@ -292,8 +292,16 @@ func (qb *FolderStore) FindMany(ctx context.Context, ids []models.FolderID) ([]*
return folders, nil
}
func (qb *FolderStore) FindByPath(ctx context.Context, p string) (*models.Folder, error) {
q := qb.selectDataset().Prepared(true).Where(qb.table().Col("path").Eq(p))
func (qb *FolderStore) FindByPath(ctx context.Context, p string, caseSensitive bool) (*models.Folder, error) {
// use like for case insensitive search
var criterion exp.BooleanExpression
if caseSensitive {
criterion = qb.table().Col("path").Eq(p)
} else {
criterion = qb.table().Col("path").ILike(p)
}
q := qb.selectDataset().Prepared(true).Where(criterion)
ret, err := qb.get(ctx, q)
if err != nil && !errors.Is(err, sql.ErrNoRows) {

6
pkg/sqlite/folder_test.go
View file

@ -89,7 +89,7 @@ func Test_FolderStore_Create(t *testing.T) {
assert.Equal(copy, s)
// ensure can find the folder
found, err := qb.FindByPath(ctx, path)
found, err := qb.FindByPath(ctx, path, true)
if err != nil {
t.Errorf("FolderStore.Find() error = %v", err)
}
@ -180,7 +180,7 @@ func Test_FolderStore_Update(t *testing.T) {
return
}
s, err := qb.FindByPath(ctx, path)
s, err := qb.FindByPath(ctx, path, true)
if err != nil {
t.Errorf("FolderStore.Find() error = %v", err)
}
@ -228,7 +228,7 @@ func Test_FolderStore_FindByPath(t *testing.T) {
for _, tt := range tests {
runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
got, err := qb.FindByPath(ctx, tt.path)
got, err := qb.FindByPath(ctx, tt.path, true)
if (err != nil) != tt.wantErr {
t.Errorf("FolderStore.FindByPath() error = %v, wantErr %v", err, tt.wantErr)
return