From 4017c42fe2fefb0f39db7cbc8efcfdc09754119a Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 2 Dec 2025 12:53:37 +1100 Subject: [PATCH] Handle modified files where the case of the filename changed on case-insensitive filesystems (#6327) * Find existing files with case insensitivity if filesystem is case insensitive * Handle case change in folders * Optimise to only test file system case sensitivity if the first query found nothing This limits the overhead to new paths, and adds an extra query for new paths to windows installs --- internal/api/resolver_query_find_file.go | 2 +- internal/api/resolver_query_find_folder.go | 2 +- internal/autotag/integration_test.go | 2 +- pkg/file/folder.go | 4 +- pkg/file/import.go | 6 +-- pkg/file/scan.go | 59 ++++++++++++++++++++-- pkg/file/video/caption.go | 2 +- pkg/gallery/import.go | 4 +- pkg/image/import.go | 2 +- pkg/models/mocks/FileReaderWriter.go | 28 +++++----- pkg/models/mocks/FolderReaderWriter.go | 14 ++--- pkg/models/repository_file.go | 4 +- pkg/models/repository_folder.go | 2 +- pkg/scene/import.go | 2 +- pkg/sqlite/file.go | 8 +-- pkg/sqlite/file_test.go | 2 +- pkg/sqlite/folder.go | 12 ++++- pkg/sqlite/folder_test.go | 6 +-- 18 files changed, 110 insertions(+), 51 deletions(-) diff --git a/internal/api/resolver_query_find_file.go b/internal/api/resolver_query_find_file.go index ae53a89b4..01c14b1ed 100644 --- a/internal/api/resolver_query_find_file.go +++ b/internal/api/resolver_query_find_file.go @@ -29,7 +29,7 @@ func (r *queryResolver) FindFile(ctx context.Context, id *string, path *string) ret = files[0] } case path != nil: - ret, err = qb.FindByPath(ctx, *path) + ret, err = qb.FindByPath(ctx, *path, true) if err == nil && ret == nil { return errors.New("file not found") } diff --git a/internal/api/resolver_query_find_folder.go b/internal/api/resolver_query_find_folder.go index a7a798dd1..d6832b7c9 100644 --- a/internal/api/resolver_query_find_folder.go +++ b/internal/api/resolver_query_find_folder.go @@ -25,7 +25,7 @@ func (r *queryResolver) FindFolder(ctx context.Context, id *string, path *string return err } case path != nil: - ret, err = qb.FindByPath(ctx, *path) + ret, err = qb.FindByPath(ctx, *path, true) if err == nil && ret == nil { return errors.New("folder not found") } diff --git a/internal/autotag/integration_test.go b/internal/autotag/integration_test.go index 565d73853..fc83df848 100644 --- a/internal/autotag/integration_test.go +++ b/internal/autotag/integration_test.go @@ -225,7 +225,7 @@ func createSceneFile(ctx context.Context, name string, folderStore models.Folder } func getOrCreateFolder(ctx context.Context, folderStore models.FolderFinderCreator, folderPath string) (*models.Folder, error) { - f, err := folderStore.FindByPath(ctx, folderPath) + f, err := folderStore.FindByPath(ctx, folderPath, true) if err != nil { return nil, fmt.Errorf("getting folder by path: %w", err) } diff --git a/pkg/file/folder.go b/pkg/file/folder.go index 451bb1d93..fe260c155 100644 --- a/pkg/file/folder.go +++ b/pkg/file/folder.go @@ -15,7 +15,9 @@ import ( // Does not create any folders in the file system func GetOrCreateFolderHierarchy(ctx context.Context, fc models.FolderFinderCreator, path string) (*models.Folder, error) { // get or create folder hierarchy - folder, err := fc.FindByPath(ctx, path) + // assume case sensitive when searching for the folder + const caseSensitive = true + folder, err := fc.FindByPath(ctx, path, caseSensitive) if err != nil { return nil, err } diff --git a/pkg/file/import.go b/pkg/file/import.go index 7c28197b8..8ca7487cb 100644 --- a/pkg/file/import.go +++ b/pkg/file/import.go @@ -120,7 +120,7 @@ func (i *Importer) baseFileJSONToBaseFile(ctx context.Context, baseJSON *jsonsch func (i *Importer) populateZipFileID(ctx context.Context, f *models.DirEntry) error { zipFilePath := i.Input.DirEntry().ZipFile if zipFilePath != "" { - zf, err := i.ReaderWriter.FindByPath(ctx, zipFilePath) + zf, err := i.ReaderWriter.FindByPath(ctx, zipFilePath, true) if err != nil { return fmt.Errorf("error finding file by path %q: %v", zipFilePath, err) } @@ -146,7 +146,7 @@ func (i *Importer) Name() string { func (i *Importer) FindExistingID(ctx context.Context) (*int, error) { path := i.Input.DirEntry().Path - existing, err := i.ReaderWriter.FindByPath(ctx, path) + existing, err := i.ReaderWriter.FindByPath(ctx, path, true) if err != nil { return nil, err } @@ -176,7 +176,7 @@ func (i *Importer) createFolderHierarchy(ctx context.Context, p string) (*models } func (i *Importer) getOrCreateFolder(ctx context.Context, path string, parent *models.Folder) (*models.Folder, error) { - folder, err := i.FolderStore.FindByPath(ctx, path) + folder, err := i.FolderStore.FindByPath(ctx, path, true) if err != nil { return nil, err } diff --git a/pkg/file/scan.go b/pkg/file/scan.go index 40f474f34..7d5a79713 100644 --- a/pkg/file/scan.go +++ b/pkg/file/scan.go @@ -443,7 +443,10 @@ func (s *scanJob) getFolderID(ctx context.Context, path string) (*models.FolderI return &v, nil } - ret, err := s.Repository.Folder.FindByPath(ctx, path) + // assume case sensitive when searching for the folder + const caseSensitive = true + + ret, err := s.Repository.Folder.FindByPath(ctx, path, caseSensitive) if err != nil { return nil, err } @@ -473,7 +476,10 @@ func (s *scanJob) getZipFileID(ctx context.Context, zipFile *scanFile) (*models. return &v, nil } - ret, err := s.Repository.File.FindByPath(ctx, path) + // assume case sensitive when searching for the zip file + const caseSensitive = true + + ret, err := s.Repository.File.FindByPath(ctx, path, caseSensitive) if err != nil { return nil, fmt.Errorf("getting zip file ID for %q: %w", path, err) } @@ -493,11 +499,26 @@ func (s *scanJob) handleFolder(ctx context.Context, file scanFile) error { defer s.incrementProgress(file) // determine if folder already exists in data store (by path) - f, err := s.Repository.Folder.FindByPath(ctx, path) + // assume case sensitive by default + f, err := s.Repository.Folder.FindByPath(ctx, path, true) if err != nil { return fmt.Errorf("checking for existing folder %q: %w", path, err) } + // #1426 / #6326 - if folder is in a case-insensitive filesystem, then try + // case insensitive searching + // assume case sensitive if in zip + if f == nil && file.ZipFileID == nil { + caseSensitive, _ := file.fs.IsPathCaseSensitive(file.Path) + + if !caseSensitive { + f, err = s.Repository.Folder.FindByPath(ctx, path, false) + if err != nil { + return fmt.Errorf("checking for existing folder %q: %w", path, err) + } + } + } + // if folder not exists, create it if f == nil { f, err = s.onNewFolder(ctx, file) @@ -611,10 +632,18 @@ func (s *scanJob) onExistingFolder(ctx context.Context, f scanFile, existing *mo // update if mod time is changed entryModTime := f.ModTime if !entryModTime.Equal(existing.ModTime) { + existing.Path = f.Path existing.ModTime = entryModTime update = true } + // #6326 - update if path has changed - should only happen if case is + // changed and filesystem is case insensitive + if existing.Path != f.Path { + existing.Path = f.Path + update = true + } + // update if zip file ID has changed fZfID := f.ZipFileID existingZfID := existing.ZipFileID @@ -647,15 +676,31 @@ func (s *scanJob) handleFile(ctx context.Context, f scanFile) error { defer s.incrementProgress(f) var ff models.File + // don't use a transaction to check if new or existing if err := s.withDB(ctx, func(ctx context.Context) error { // determine if file already exists in data store + // assume case sensitive when searching for the file to begin with var err error - ff, err = s.Repository.File.FindByPath(ctx, f.Path) + ff, err = s.Repository.File.FindByPath(ctx, f.Path, true) if err != nil { return fmt.Errorf("checking for existing file %q: %w", f.Path, err) } + // #1426 / #6326 - if file is in a case-insensitive filesystem, then try + // case insensitive search + // assume case sensitive if in zip + if ff == nil && f.ZipFileID != nil { + caseSensitive, _ := f.fs.IsPathCaseSensitive(f.Path) + + if !caseSensitive { + ff, err = s.Repository.File.FindByPath(ctx, f.Path, false) + if err != nil { + return fmt.Errorf("checking for existing file %q: %w", f.Path, err) + } + } + } + if ff == nil { // returns a file only if it is actually new ff, err = s.onNewFile(ctx, f) @@ -879,6 +924,7 @@ func (s *scanJob) handleRename(ctx context.Context, f models.File, fp []models.F // #1426 - if file exists but is a case-insensitive match for the // original filename, and the filesystem is case-insensitive // then treat it as a move + // #6326 - this should now be handled earlier, and this shouldn't be necessary if caseSensitive, _ := fs.IsPathCaseSensitive(other.Base().Path); !caseSensitive { // treat as a move missing = append(missing, other) @@ -1026,7 +1072,8 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing model path := base.Path fileModTime := f.ModTime - updated := !fileModTime.Equal(base.ModTime) + // #6326 - also force a rescan if the basename changed + updated := !fileModTime.Equal(base.ModTime) || base.Basename != f.Basename forceRescan := s.options.Rescan if !updated && !forceRescan { @@ -1041,6 +1088,8 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing model logger.Infof("%s has been updated: rescanning", path) } + // #6326 - update basename in case it changed + base.Basename = f.Basename base.ModTime = fileModTime base.Size = f.Size base.UpdatedAt = time.Now() diff --git a/pkg/file/video/caption.go b/pkg/file/video/caption.go index bec3db6fd..43723864f 100644 --- a/pkg/file/video/caption.go +++ b/pkg/file/video/caption.go @@ -97,7 +97,7 @@ func AssociateCaptions(ctx context.Context, captionPath string, txnMgr txn.Manag captionPrefix := getCaptionPrefix(captionPath) if err := txn.WithTxn(ctx, txnMgr, func(ctx context.Context) error { var err error - files, er := fqb.FindAllByPath(ctx, captionPrefix+"*") + files, er := fqb.FindAllByPath(ctx, captionPrefix+"*", true) if er != nil { return fmt.Errorf("searching for scene %s: %w", captionPrefix, er) diff --git a/pkg/gallery/import.go b/pkg/gallery/import.go index 7cdf53691..0068b3f1c 100644 --- a/pkg/gallery/import.go +++ b/pkg/gallery/import.go @@ -265,7 +265,7 @@ func (i *Importer) populateFilesFolder(ctx context.Context) error { for _, ref := range i.Input.ZipFiles { path := ref - f, err := i.FileFinder.FindByPath(ctx, path) + f, err := i.FileFinder.FindByPath(ctx, path, true) if err != nil { return fmt.Errorf("error finding file: %w", err) } @@ -281,7 +281,7 @@ func (i *Importer) populateFilesFolder(ctx context.Context) error { if i.Input.FolderPath != "" { path := i.Input.FolderPath - f, err := i.FolderFinder.FindByPath(ctx, path) + f, err := i.FolderFinder.FindByPath(ctx, path, true) if err != nil { return fmt.Errorf("error finding folder: %w", err) } diff --git a/pkg/image/import.go b/pkg/image/import.go index ec200af04..bf92a6ae8 100644 --- a/pkg/image/import.go +++ b/pkg/image/import.go @@ -110,7 +110,7 @@ func (i *Importer) populateFiles(ctx context.Context) error { for _, ref := range i.Input.Files { path := ref - f, err := i.FileFinder.FindByPath(ctx, path) + f, err := i.FileFinder.FindByPath(ctx, path, true) if err != nil { return fmt.Errorf("error finding file: %w", err) } diff --git a/pkg/models/mocks/FileReaderWriter.go b/pkg/models/mocks/FileReaderWriter.go index 12a1b3075..97a0136e6 100644 --- a/pkg/models/mocks/FileReaderWriter.go +++ b/pkg/models/mocks/FileReaderWriter.go @@ -130,13 +130,13 @@ func (_m *FileReaderWriter) Find(ctx context.Context, id ...models.FileID) ([]mo return r0, r1 } -// FindAllByPath provides a mock function with given fields: ctx, path -func (_m *FileReaderWriter) FindAllByPath(ctx context.Context, path string) ([]models.File, error) { - ret := _m.Called(ctx, path) +// FindAllByPath provides a mock function with given fields: ctx, path, caseSensitive +func (_m *FileReaderWriter) FindAllByPath(ctx context.Context, path string, caseSensitive bool) ([]models.File, error) { + ret := _m.Called(ctx, path, caseSensitive) var r0 []models.File - if rf, ok := ret.Get(0).(func(context.Context, string) []models.File); ok { - r0 = rf(ctx, path) + if rf, ok := ret.Get(0).(func(context.Context, string, bool) []models.File); ok { + r0 = rf(ctx, path, caseSensitive) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]models.File) @@ -144,8 +144,8 @@ func (_m *FileReaderWriter) FindAllByPath(ctx context.Context, path string) ([]m } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, path) + if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok { + r1 = rf(ctx, path, caseSensitive) } else { r1 = ret.Error(1) } @@ -222,13 +222,13 @@ func (_m *FileReaderWriter) FindByFingerprint(ctx context.Context, fp models.Fin return r0, r1 } -// FindByPath provides a mock function with given fields: ctx, path -func (_m *FileReaderWriter) FindByPath(ctx context.Context, path string) (models.File, error) { - ret := _m.Called(ctx, path) +// FindByPath provides a mock function with given fields: ctx, path, caseSensitive +func (_m *FileReaderWriter) FindByPath(ctx context.Context, path string, caseSensitive bool) (models.File, error) { + ret := _m.Called(ctx, path, caseSensitive) var r0 models.File - if rf, ok := ret.Get(0).(func(context.Context, string) models.File); ok { - r0 = rf(ctx, path) + if rf, ok := ret.Get(0).(func(context.Context, string, bool) models.File); ok { + r0 = rf(ctx, path, caseSensitive) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(models.File) @@ -236,8 +236,8 @@ func (_m *FileReaderWriter) FindByPath(ctx context.Context, path string) (models } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, path) + if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok { + r1 = rf(ctx, path, caseSensitive) } else { r1 = ret.Error(1) } diff --git a/pkg/models/mocks/FolderReaderWriter.go b/pkg/models/mocks/FolderReaderWriter.go index 512925fd6..7bca013fe 100644 --- a/pkg/models/mocks/FolderReaderWriter.go +++ b/pkg/models/mocks/FolderReaderWriter.go @@ -132,13 +132,13 @@ func (_m *FolderReaderWriter) FindByParentFolderID(ctx context.Context, parentFo return r0, r1 } -// FindByPath provides a mock function with given fields: ctx, path -func (_m *FolderReaderWriter) FindByPath(ctx context.Context, path string) (*models.Folder, error) { - ret := _m.Called(ctx, path) +// FindByPath provides a mock function with given fields: ctx, path, caseSensitive +func (_m *FolderReaderWriter) FindByPath(ctx context.Context, path string, caseSensitive bool) (*models.Folder, error) { + ret := _m.Called(ctx, path, caseSensitive) var r0 *models.Folder - if rf, ok := ret.Get(0).(func(context.Context, string) *models.Folder); ok { - r0 = rf(ctx, path) + if rf, ok := ret.Get(0).(func(context.Context, string, bool) *models.Folder); ok { + r0 = rf(ctx, path, caseSensitive) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*models.Folder) @@ -146,8 +146,8 @@ func (_m *FolderReaderWriter) FindByPath(ctx context.Context, path string) (*mod } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, path) + if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok { + r1 = rf(ctx, path, caseSensitive) } else { r1 = ret.Error(1) } diff --git a/pkg/models/repository_file.go b/pkg/models/repository_file.go index 0819b25a5..c851ce08c 100644 --- a/pkg/models/repository_file.go +++ b/pkg/models/repository_file.go @@ -13,9 +13,9 @@ type FileGetter interface { // FileFinder provides methods to find files. type FileFinder interface { FileGetter - FindAllByPath(ctx context.Context, path string) ([]File, error) + FindAllByPath(ctx context.Context, path string, caseSensitive bool) ([]File, error) FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]File, error) - FindByPath(ctx context.Context, path string) (File, error) + FindByPath(ctx context.Context, path string, caseSensitive bool) (File, error) FindByFingerprint(ctx context.Context, fp Fingerprint) ([]File, error) FindByZipFileID(ctx context.Context, zipFileID FileID) ([]File, error) FindByFileInfo(ctx context.Context, info fs.FileInfo, size int64) ([]File, error) diff --git a/pkg/models/repository_folder.go b/pkg/models/repository_folder.go index 671e8780d..3d0fdb822 100644 --- a/pkg/models/repository_folder.go +++ b/pkg/models/repository_folder.go @@ -12,7 +12,7 @@ type FolderGetter interface { type FolderFinder interface { FolderGetter FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]*Folder, error) - FindByPath(ctx context.Context, path string) (*Folder, error) + FindByPath(ctx context.Context, path string, caseSensitive bool) (*Folder, error) FindByZipFileID(ctx context.Context, zipFileID FileID) ([]*Folder, error) FindByParentFolderID(ctx context.Context, parentFolderID FolderID) ([]*Folder, error) } diff --git a/pkg/scene/import.go b/pkg/scene/import.go index e1248a77c..efffd380d 100644 --- a/pkg/scene/import.go +++ b/pkg/scene/import.go @@ -164,7 +164,7 @@ func (i *Importer) populateFiles(ctx context.Context) error { for _, ref := range i.Input.Files { path := ref - f, err := i.FileFinder.FindByPath(ctx, path) + f, err := i.FileFinder.FindByPath(ctx, path, true) if err != nil { return fmt.Errorf("error finding file: %w", err) } diff --git a/pkg/sqlite/file.go b/pkg/sqlite/file.go index 2144356c5..1be5648b4 100644 --- a/pkg/sqlite/file.go +++ b/pkg/sqlite/file.go @@ -625,9 +625,9 @@ func (qb *FileStore) find(ctx context.Context, id models.FileID) (models.File, e } // FindByPath returns the first file that matches the given path. Wildcard characters are supported. -func (qb *FileStore) FindByPath(ctx context.Context, p string) (models.File, error) { +func (qb *FileStore) FindByPath(ctx context.Context, p string, caseSensitive bool) (models.File, error) { - ret, err := qb.FindAllByPath(ctx, p) + ret, err := qb.FindAllByPath(ctx, p, caseSensitive) if err != nil { return nil, err @@ -642,7 +642,7 @@ func (qb *FileStore) FindByPath(ctx context.Context, p string) (models.File, err // FindAllByPath returns all the files that match the given path. // Wildcard characters are supported. -func (qb *FileStore) FindAllByPath(ctx context.Context, p string) ([]models.File, error) { +func (qb *FileStore) FindAllByPath(ctx context.Context, p string, caseSensitive bool) ([]models.File, error) { // separate basename from path basename := filepath.Base(p) dirName := filepath.Dir(p) @@ -657,7 +657,7 @@ func (qb *FileStore) FindAllByPath(ctx context.Context, p string) ([]models.File // like uses case-insensitive matching. Only use like if wildcards are used q := qb.selectDataset().Prepared(true) - if strings.Contains(basename, "%") || strings.Contains(dirName, "%") { + if strings.Contains(basename, "%") || strings.Contains(dirName, "%") || !caseSensitive { q = q.Where( folderTable.Col("path").Like(dirName), table.Col("basename").Like(basename), diff --git a/pkg/sqlite/file_test.go b/pkg/sqlite/file_test.go index 766ffcc70..8422390c0 100644 --- a/pkg/sqlite/file_test.go +++ b/pkg/sqlite/file_test.go @@ -551,7 +551,7 @@ func Test_FileStore_FindByPath(t *testing.T) { for _, tt := range tests { runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) { assert := assert.New(t) - got, err := qb.FindByPath(ctx, tt.path) + got, err := qb.FindByPath(ctx, tt.path, true) if (err != nil) != tt.wantErr { t.Errorf("FileStore.FindByPath() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/sqlite/folder.go b/pkg/sqlite/folder.go index 3ac962b8b..f250f7861 100644 --- a/pkg/sqlite/folder.go +++ b/pkg/sqlite/folder.go @@ -292,8 +292,16 @@ func (qb *FolderStore) FindMany(ctx context.Context, ids []models.FolderID) ([]* return folders, nil } -func (qb *FolderStore) FindByPath(ctx context.Context, p string) (*models.Folder, error) { - q := qb.selectDataset().Prepared(true).Where(qb.table().Col("path").Eq(p)) +func (qb *FolderStore) FindByPath(ctx context.Context, p string, caseSensitive bool) (*models.Folder, error) { + // use like for case insensitive search + var criterion exp.BooleanExpression + if caseSensitive { + criterion = qb.table().Col("path").Eq(p) + } else { + criterion = qb.table().Col("path").ILike(p) + } + + q := qb.selectDataset().Prepared(true).Where(criterion) ret, err := qb.get(ctx, q) if err != nil && !errors.Is(err, sql.ErrNoRows) { diff --git a/pkg/sqlite/folder_test.go b/pkg/sqlite/folder_test.go index 1d948d063..15b2b96b8 100644 --- a/pkg/sqlite/folder_test.go +++ b/pkg/sqlite/folder_test.go @@ -89,7 +89,7 @@ func Test_FolderStore_Create(t *testing.T) { assert.Equal(copy, s) // ensure can find the folder - found, err := qb.FindByPath(ctx, path) + found, err := qb.FindByPath(ctx, path, true) if err != nil { t.Errorf("FolderStore.Find() error = %v", err) } @@ -180,7 +180,7 @@ func Test_FolderStore_Update(t *testing.T) { return } - s, err := qb.FindByPath(ctx, path) + s, err := qb.FindByPath(ctx, path, true) if err != nil { t.Errorf("FolderStore.Find() error = %v", err) } @@ -228,7 +228,7 @@ func Test_FolderStore_FindByPath(t *testing.T) { for _, tt := range tests { runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) { - got, err := qb.FindByPath(ctx, tt.path) + got, err := qb.FindByPath(ctx, tt.path, true) if (err != nil) != tt.wantErr { t.Errorf("FolderStore.FindByPath() error = %v, wantErr %v", err, tt.wantErr) return