diff --git a/internal/api/resolver_query_find_file.go b/internal/api/resolver_query_find_file.go index ae53a89b4..01c14b1ed 100644 --- a/internal/api/resolver_query_find_file.go +++ b/internal/api/resolver_query_find_file.go @@ -29,7 +29,7 @@ func (r *queryResolver) FindFile(ctx context.Context, id *string, path *string) ret = files[0] } case path != nil: - ret, err = qb.FindByPath(ctx, *path) + ret, err = qb.FindByPath(ctx, *path, true) if err == nil && ret == nil { return errors.New("file not found") } diff --git a/internal/api/resolver_query_find_folder.go b/internal/api/resolver_query_find_folder.go index a7a798dd1..d6832b7c9 100644 --- a/internal/api/resolver_query_find_folder.go +++ b/internal/api/resolver_query_find_folder.go @@ -25,7 +25,7 @@ func (r *queryResolver) FindFolder(ctx context.Context, id *string, path *string return err } case path != nil: - ret, err = qb.FindByPath(ctx, *path) + ret, err = qb.FindByPath(ctx, *path, true) if err == nil && ret == nil { return errors.New("folder not found") } diff --git a/internal/autotag/integration_test.go b/internal/autotag/integration_test.go index 565d73853..fc83df848 100644 --- a/internal/autotag/integration_test.go +++ b/internal/autotag/integration_test.go @@ -225,7 +225,7 @@ func createSceneFile(ctx context.Context, name string, folderStore models.Folder } func getOrCreateFolder(ctx context.Context, folderStore models.FolderFinderCreator, folderPath string) (*models.Folder, error) { - f, err := folderStore.FindByPath(ctx, folderPath) + f, err := folderStore.FindByPath(ctx, folderPath, true) if err != nil { return nil, fmt.Errorf("getting folder by path: %w", err) } diff --git a/pkg/file/folder.go b/pkg/file/folder.go index 451bb1d93..fe260c155 100644 --- a/pkg/file/folder.go +++ b/pkg/file/folder.go @@ -15,7 +15,9 @@ import ( // Does not create any folders in the file system func GetOrCreateFolderHierarchy(ctx context.Context, fc models.FolderFinderCreator, path string) (*models.Folder, error) { // get or create folder hierarchy - folder, err := fc.FindByPath(ctx, path) + // assume case sensitive when searching for the folder + const caseSensitive = true + folder, err := fc.FindByPath(ctx, path, caseSensitive) if err != nil { return nil, err } diff --git a/pkg/file/import.go b/pkg/file/import.go index 7c28197b8..8ca7487cb 100644 --- a/pkg/file/import.go +++ b/pkg/file/import.go @@ -120,7 +120,7 @@ func (i *Importer) baseFileJSONToBaseFile(ctx context.Context, baseJSON *jsonsch func (i *Importer) populateZipFileID(ctx context.Context, f *models.DirEntry) error { zipFilePath := i.Input.DirEntry().ZipFile if zipFilePath != "" { - zf, err := i.ReaderWriter.FindByPath(ctx, zipFilePath) + zf, err := i.ReaderWriter.FindByPath(ctx, zipFilePath, true) if err != nil { return fmt.Errorf("error finding file by path %q: %v", zipFilePath, err) } @@ -146,7 +146,7 @@ func (i *Importer) Name() string { func (i *Importer) FindExistingID(ctx context.Context) (*int, error) { path := i.Input.DirEntry().Path - existing, err := i.ReaderWriter.FindByPath(ctx, path) + existing, err := i.ReaderWriter.FindByPath(ctx, path, true) if err != nil { return nil, err } @@ -176,7 +176,7 @@ func (i *Importer) createFolderHierarchy(ctx context.Context, p string) (*models } func (i *Importer) getOrCreateFolder(ctx context.Context, path string, parent *models.Folder) (*models.Folder, error) { - folder, err := i.FolderStore.FindByPath(ctx, path) + folder, err := i.FolderStore.FindByPath(ctx, path, true) if err != nil { return nil, err } diff --git a/pkg/file/scan.go b/pkg/file/scan.go index 40f474f34..7d5a79713 100644 --- a/pkg/file/scan.go +++ b/pkg/file/scan.go @@ -443,7 +443,10 @@ func (s *scanJob) getFolderID(ctx context.Context, path string) (*models.FolderI return &v, nil } - ret, err := s.Repository.Folder.FindByPath(ctx, path) + // assume case sensitive when searching for the folder + const caseSensitive = true + + ret, err := s.Repository.Folder.FindByPath(ctx, path, caseSensitive) if err != nil { return nil, err } @@ -473,7 +476,10 @@ func (s *scanJob) getZipFileID(ctx context.Context, zipFile *scanFile) (*models. return &v, nil } - ret, err := s.Repository.File.FindByPath(ctx, path) + // assume case sensitive when searching for the zip file + const caseSensitive = true + + ret, err := s.Repository.File.FindByPath(ctx, path, caseSensitive) if err != nil { return nil, fmt.Errorf("getting zip file ID for %q: %w", path, err) } @@ -493,11 +499,26 @@ func (s *scanJob) handleFolder(ctx context.Context, file scanFile) error { defer s.incrementProgress(file) // determine if folder already exists in data store (by path) - f, err := s.Repository.Folder.FindByPath(ctx, path) + // assume case sensitive by default + f, err := s.Repository.Folder.FindByPath(ctx, path, true) if err != nil { return fmt.Errorf("checking for existing folder %q: %w", path, err) } + // #1426 / #6326 - if folder is in a case-insensitive filesystem, then try + // case insensitive searching + // assume case sensitive if in zip + if f == nil && file.ZipFileID == nil { + caseSensitive, _ := file.fs.IsPathCaseSensitive(file.Path) + + if !caseSensitive { + f, err = s.Repository.Folder.FindByPath(ctx, path, false) + if err != nil { + return fmt.Errorf("checking for existing folder %q: %w", path, err) + } + } + } + // if folder not exists, create it if f == nil { f, err = s.onNewFolder(ctx, file) @@ -611,10 +632,18 @@ func (s *scanJob) onExistingFolder(ctx context.Context, f scanFile, existing *mo // update if mod time is changed entryModTime := f.ModTime if !entryModTime.Equal(existing.ModTime) { + existing.Path = f.Path existing.ModTime = entryModTime update = true } + // #6326 - update if path has changed - should only happen if case is + // changed and filesystem is case insensitive + if existing.Path != f.Path { + existing.Path = f.Path + update = true + } + // update if zip file ID has changed fZfID := f.ZipFileID existingZfID := existing.ZipFileID @@ -647,15 +676,31 @@ func (s *scanJob) handleFile(ctx context.Context, f scanFile) error { defer s.incrementProgress(f) var ff models.File + // don't use a transaction to check if new or existing if err := s.withDB(ctx, func(ctx context.Context) error { // determine if file already exists in data store + // assume case sensitive when searching for the file to begin with var err error - ff, err = s.Repository.File.FindByPath(ctx, f.Path) + ff, err = s.Repository.File.FindByPath(ctx, f.Path, true) if err != nil { return fmt.Errorf("checking for existing file %q: %w", f.Path, err) } + // #1426 / #6326 - if file is in a case-insensitive filesystem, then try + // case insensitive search + // assume case sensitive if in zip + if ff == nil && f.ZipFileID != nil { + caseSensitive, _ := f.fs.IsPathCaseSensitive(f.Path) + + if !caseSensitive { + ff, err = s.Repository.File.FindByPath(ctx, f.Path, false) + if err != nil { + return fmt.Errorf("checking for existing file %q: %w", f.Path, err) + } + } + } + if ff == nil { // returns a file only if it is actually new ff, err = s.onNewFile(ctx, f) @@ -879,6 +924,7 @@ func (s *scanJob) handleRename(ctx context.Context, f models.File, fp []models.F // #1426 - if file exists but is a case-insensitive match for the // original filename, and the filesystem is case-insensitive // then treat it as a move + // #6326 - this should now be handled earlier, and this shouldn't be necessary if caseSensitive, _ := fs.IsPathCaseSensitive(other.Base().Path); !caseSensitive { // treat as a move missing = append(missing, other) @@ -1026,7 +1072,8 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing model path := base.Path fileModTime := f.ModTime - updated := !fileModTime.Equal(base.ModTime) + // #6326 - also force a rescan if the basename changed + updated := !fileModTime.Equal(base.ModTime) || base.Basename != f.Basename forceRescan := s.options.Rescan if !updated && !forceRescan { @@ -1041,6 +1088,8 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing model logger.Infof("%s has been updated: rescanning", path) } + // #6326 - update basename in case it changed + base.Basename = f.Basename base.ModTime = fileModTime base.Size = f.Size base.UpdatedAt = time.Now() diff --git a/pkg/file/video/caption.go b/pkg/file/video/caption.go index bec3db6fd..43723864f 100644 --- a/pkg/file/video/caption.go +++ b/pkg/file/video/caption.go @@ -97,7 +97,7 @@ func AssociateCaptions(ctx context.Context, captionPath string, txnMgr txn.Manag captionPrefix := getCaptionPrefix(captionPath) if err := txn.WithTxn(ctx, txnMgr, func(ctx context.Context) error { var err error - files, er := fqb.FindAllByPath(ctx, captionPrefix+"*") + files, er := fqb.FindAllByPath(ctx, captionPrefix+"*", true) if er != nil { return fmt.Errorf("searching for scene %s: %w", captionPrefix, er) diff --git a/pkg/gallery/import.go b/pkg/gallery/import.go index 7cdf53691..0068b3f1c 100644 --- a/pkg/gallery/import.go +++ b/pkg/gallery/import.go @@ -265,7 +265,7 @@ func (i *Importer) populateFilesFolder(ctx context.Context) error { for _, ref := range i.Input.ZipFiles { path := ref - f, err := i.FileFinder.FindByPath(ctx, path) + f, err := i.FileFinder.FindByPath(ctx, path, true) if err != nil { return fmt.Errorf("error finding file: %w", err) } @@ -281,7 +281,7 @@ func (i *Importer) populateFilesFolder(ctx context.Context) error { if i.Input.FolderPath != "" { path := i.Input.FolderPath - f, err := i.FolderFinder.FindByPath(ctx, path) + f, err := i.FolderFinder.FindByPath(ctx, path, true) if err != nil { return fmt.Errorf("error finding folder: %w", err) } diff --git a/pkg/image/import.go b/pkg/image/import.go index ec200af04..bf92a6ae8 100644 --- a/pkg/image/import.go +++ b/pkg/image/import.go @@ -110,7 +110,7 @@ func (i *Importer) populateFiles(ctx context.Context) error { for _, ref := range i.Input.Files { path := ref - f, err := i.FileFinder.FindByPath(ctx, path) + f, err := i.FileFinder.FindByPath(ctx, path, true) if err != nil { return fmt.Errorf("error finding file: %w", err) } diff --git a/pkg/models/mocks/FileReaderWriter.go b/pkg/models/mocks/FileReaderWriter.go index 12a1b3075..97a0136e6 100644 --- a/pkg/models/mocks/FileReaderWriter.go +++ b/pkg/models/mocks/FileReaderWriter.go @@ -130,13 +130,13 @@ func (_m *FileReaderWriter) Find(ctx context.Context, id ...models.FileID) ([]mo return r0, r1 } -// FindAllByPath provides a mock function with given fields: ctx, path -func (_m *FileReaderWriter) FindAllByPath(ctx context.Context, path string) ([]models.File, error) { - ret := _m.Called(ctx, path) +// FindAllByPath provides a mock function with given fields: ctx, path, caseSensitive +func (_m *FileReaderWriter) FindAllByPath(ctx context.Context, path string, caseSensitive bool) ([]models.File, error) { + ret := _m.Called(ctx, path, caseSensitive) var r0 []models.File - if rf, ok := ret.Get(0).(func(context.Context, string) []models.File); ok { - r0 = rf(ctx, path) + if rf, ok := ret.Get(0).(func(context.Context, string, bool) []models.File); ok { + r0 = rf(ctx, path, caseSensitive) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]models.File) @@ -144,8 +144,8 @@ func (_m *FileReaderWriter) FindAllByPath(ctx context.Context, path string) ([]m } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, path) + if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok { + r1 = rf(ctx, path, caseSensitive) } else { r1 = ret.Error(1) } @@ -222,13 +222,13 @@ func (_m *FileReaderWriter) FindByFingerprint(ctx context.Context, fp models.Fin return r0, r1 } -// FindByPath provides a mock function with given fields: ctx, path -func (_m *FileReaderWriter) FindByPath(ctx context.Context, path string) (models.File, error) { - ret := _m.Called(ctx, path) +// FindByPath provides a mock function with given fields: ctx, path, caseSensitive +func (_m *FileReaderWriter) FindByPath(ctx context.Context, path string, caseSensitive bool) (models.File, error) { + ret := _m.Called(ctx, path, caseSensitive) var r0 models.File - if rf, ok := ret.Get(0).(func(context.Context, string) models.File); ok { - r0 = rf(ctx, path) + if rf, ok := ret.Get(0).(func(context.Context, string, bool) models.File); ok { + r0 = rf(ctx, path, caseSensitive) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(models.File) @@ -236,8 +236,8 @@ func (_m *FileReaderWriter) FindByPath(ctx context.Context, path string) (models } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, path) + if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok { + r1 = rf(ctx, path, caseSensitive) } else { r1 = ret.Error(1) } diff --git a/pkg/models/mocks/FolderReaderWriter.go b/pkg/models/mocks/FolderReaderWriter.go index 512925fd6..7bca013fe 100644 --- a/pkg/models/mocks/FolderReaderWriter.go +++ b/pkg/models/mocks/FolderReaderWriter.go @@ -132,13 +132,13 @@ func (_m *FolderReaderWriter) FindByParentFolderID(ctx context.Context, parentFo return r0, r1 } -// FindByPath provides a mock function with given fields: ctx, path -func (_m *FolderReaderWriter) FindByPath(ctx context.Context, path string) (*models.Folder, error) { - ret := _m.Called(ctx, path) +// FindByPath provides a mock function with given fields: ctx, path, caseSensitive +func (_m *FolderReaderWriter) FindByPath(ctx context.Context, path string, caseSensitive bool) (*models.Folder, error) { + ret := _m.Called(ctx, path, caseSensitive) var r0 *models.Folder - if rf, ok := ret.Get(0).(func(context.Context, string) *models.Folder); ok { - r0 = rf(ctx, path) + if rf, ok := ret.Get(0).(func(context.Context, string, bool) *models.Folder); ok { + r0 = rf(ctx, path, caseSensitive) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*models.Folder) @@ -146,8 +146,8 @@ func (_m *FolderReaderWriter) FindByPath(ctx context.Context, path string) (*mod } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, path) + if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok { + r1 = rf(ctx, path, caseSensitive) } else { r1 = ret.Error(1) } diff --git a/pkg/models/repository_file.go b/pkg/models/repository_file.go index 0819b25a5..c851ce08c 100644 --- a/pkg/models/repository_file.go +++ b/pkg/models/repository_file.go @@ -13,9 +13,9 @@ type FileGetter interface { // FileFinder provides methods to find files. type FileFinder interface { FileGetter - FindAllByPath(ctx context.Context, path string) ([]File, error) + FindAllByPath(ctx context.Context, path string, caseSensitive bool) ([]File, error) FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]File, error) - FindByPath(ctx context.Context, path string) (File, error) + FindByPath(ctx context.Context, path string, caseSensitive bool) (File, error) FindByFingerprint(ctx context.Context, fp Fingerprint) ([]File, error) FindByZipFileID(ctx context.Context, zipFileID FileID) ([]File, error) FindByFileInfo(ctx context.Context, info fs.FileInfo, size int64) ([]File, error) diff --git a/pkg/models/repository_folder.go b/pkg/models/repository_folder.go index 671e8780d..3d0fdb822 100644 --- a/pkg/models/repository_folder.go +++ b/pkg/models/repository_folder.go @@ -12,7 +12,7 @@ type FolderGetter interface { type FolderFinder interface { FolderGetter FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]*Folder, error) - FindByPath(ctx context.Context, path string) (*Folder, error) + FindByPath(ctx context.Context, path string, caseSensitive bool) (*Folder, error) FindByZipFileID(ctx context.Context, zipFileID FileID) ([]*Folder, error) FindByParentFolderID(ctx context.Context, parentFolderID FolderID) ([]*Folder, error) } diff --git a/pkg/scene/import.go b/pkg/scene/import.go index e1248a77c..efffd380d 100644 --- a/pkg/scene/import.go +++ b/pkg/scene/import.go @@ -164,7 +164,7 @@ func (i *Importer) populateFiles(ctx context.Context) error { for _, ref := range i.Input.Files { path := ref - f, err := i.FileFinder.FindByPath(ctx, path) + f, err := i.FileFinder.FindByPath(ctx, path, true) if err != nil { return fmt.Errorf("error finding file: %w", err) } diff --git a/pkg/sqlite/file.go b/pkg/sqlite/file.go index 2144356c5..1be5648b4 100644 --- a/pkg/sqlite/file.go +++ b/pkg/sqlite/file.go @@ -625,9 +625,9 @@ func (qb *FileStore) find(ctx context.Context, id models.FileID) (models.File, e } // FindByPath returns the first file that matches the given path. Wildcard characters are supported. -func (qb *FileStore) FindByPath(ctx context.Context, p string) (models.File, error) { +func (qb *FileStore) FindByPath(ctx context.Context, p string, caseSensitive bool) (models.File, error) { - ret, err := qb.FindAllByPath(ctx, p) + ret, err := qb.FindAllByPath(ctx, p, caseSensitive) if err != nil { return nil, err @@ -642,7 +642,7 @@ func (qb *FileStore) FindByPath(ctx context.Context, p string) (models.File, err // FindAllByPath returns all the files that match the given path. // Wildcard characters are supported. -func (qb *FileStore) FindAllByPath(ctx context.Context, p string) ([]models.File, error) { +func (qb *FileStore) FindAllByPath(ctx context.Context, p string, caseSensitive bool) ([]models.File, error) { // separate basename from path basename := filepath.Base(p) dirName := filepath.Dir(p) @@ -657,7 +657,7 @@ func (qb *FileStore) FindAllByPath(ctx context.Context, p string) ([]models.File // like uses case-insensitive matching. Only use like if wildcards are used q := qb.selectDataset().Prepared(true) - if strings.Contains(basename, "%") || strings.Contains(dirName, "%") { + if strings.Contains(basename, "%") || strings.Contains(dirName, "%") || !caseSensitive { q = q.Where( folderTable.Col("path").Like(dirName), table.Col("basename").Like(basename), diff --git a/pkg/sqlite/file_test.go b/pkg/sqlite/file_test.go index 766ffcc70..8422390c0 100644 --- a/pkg/sqlite/file_test.go +++ b/pkg/sqlite/file_test.go @@ -551,7 +551,7 @@ func Test_FileStore_FindByPath(t *testing.T) { for _, tt := range tests { runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) { assert := assert.New(t) - got, err := qb.FindByPath(ctx, tt.path) + got, err := qb.FindByPath(ctx, tt.path, true) if (err != nil) != tt.wantErr { t.Errorf("FileStore.FindByPath() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/sqlite/folder.go b/pkg/sqlite/folder.go index 3ac962b8b..f250f7861 100644 --- a/pkg/sqlite/folder.go +++ b/pkg/sqlite/folder.go @@ -292,8 +292,16 @@ func (qb *FolderStore) FindMany(ctx context.Context, ids []models.FolderID) ([]* return folders, nil } -func (qb *FolderStore) FindByPath(ctx context.Context, p string) (*models.Folder, error) { - q := qb.selectDataset().Prepared(true).Where(qb.table().Col("path").Eq(p)) +func (qb *FolderStore) FindByPath(ctx context.Context, p string, caseSensitive bool) (*models.Folder, error) { + // use like for case insensitive search + var criterion exp.BooleanExpression + if caseSensitive { + criterion = qb.table().Col("path").Eq(p) + } else { + criterion = qb.table().Col("path").ILike(p) + } + + q := qb.selectDataset().Prepared(true).Where(criterion) ret, err := qb.get(ctx, q) if err != nil && !errors.Is(err, sql.ErrNoRows) { diff --git a/pkg/sqlite/folder_test.go b/pkg/sqlite/folder_test.go index 1d948d063..15b2b96b8 100644 --- a/pkg/sqlite/folder_test.go +++ b/pkg/sqlite/folder_test.go @@ -89,7 +89,7 @@ func Test_FolderStore_Create(t *testing.T) { assert.Equal(copy, s) // ensure can find the folder - found, err := qb.FindByPath(ctx, path) + found, err := qb.FindByPath(ctx, path, true) if err != nil { t.Errorf("FolderStore.Find() error = %v", err) } @@ -180,7 +180,7 @@ func Test_FolderStore_Update(t *testing.T) { return } - s, err := qb.FindByPath(ctx, path) + s, err := qb.FindByPath(ctx, path, true) if err != nil { t.Errorf("FolderStore.Find() error = %v", err) } @@ -228,7 +228,7 @@ func Test_FolderStore_FindByPath(t *testing.T) { for _, tt := range tests { runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) { - got, err := qb.FindByPath(ctx, tt.path) + got, err := qb.FindByPath(ctx, tt.path, true) if (err != nil) != tt.wantErr { t.Errorf("FolderStore.FindByPath() error = %v, wantErr %v", err, tt.wantErr) return