[Files Refactor] Performance tuning (#2784)

* Improve image query performance
* Tune queries
* Fix db generator
* Don't show release notes in setup
* Further tune indexes
* Log when creating screenshot

internal/autotag/integration_test.go

@@ -27,8 +27,8 @@ const testName = "Foo's Bar"
 const existingStudioName = "ExistingStudio"
 
 const existingStudioSceneName = testName + ".dontChangeStudio.mp4"
-const existingStudioImageName = testName + ".dontChangeStudio.mp4"
+const existingStudioImageName = testName + ".dontChangeStudio.png"
-const existingStudioGalleryName = testName + ".dontChangeStudio.mp4"
+const existingStudioGalleryName = testName + ".dontChangeStudio.zip"
 
 var existingStudioID int
 

pkg/file/clean.go

@@ -47,7 +47,7 @@ func (s *Cleaner) Clean(ctx context.Context, options CleanOptions, progress *job
 	}
 
 	if err := j.execute(ctx); err != nil {
-		logger.Errorf("error cleaning files: %w", err)
+		logger.Errorf("error cleaning files: %v", err)
 		return
 	}
 }

pkg/models/image.go

@@ -97,7 +97,6 @@ type ImageReader interface {
 	FindByChecksum(ctx context.Context, checksum string) ([]*Image, error)
 	FindByGalleryID(ctx context.Context, galleryID int) ([]*Image, error)
 	CountByGalleryID(ctx context.Context, galleryID int) (int, error)
-	FindByPath(ctx context.Context, path string) ([]*Image, error)
 	Count(ctx context.Context) (int, error)
 	Size(ctx context.Context) (float64, error)
 	All(ctx context.Context) ([]*Image, error)

pkg/scene/generate/screenshot.go

@@ -32,6 +32,8 @@ func (g Generator) Screenshot(ctx context.Context, input string, hash string, vi
 		}
 	}
 
+	logger.Infof("Creating screenshot for %s", input)
+
 	at := screenshotDurationProportion * videoDuration
 	if options.At != nil {
 		at = *options.At

pkg/sqlite/file.go

@@ -738,7 +738,7 @@ func (qb *FileStore) makeFilter(ctx context.Context, fileFilter *models.FileFilt
 		query.not(qb.makeFilter(ctx, fileFilter.Not))
 	}
 
-	query.handleCriterion(ctx, pathCriterionHandler(fileFilter.Path, "folders.path", "files.basename"))
+	query.handleCriterion(ctx, pathCriterionHandler(fileFilter.Path, "folders.path", "files.basename", nil))
 
 	return query
 }

pkg/sqlite/file_test.go

@@ -155,7 +155,7 @@ func Test_fileFileStore_Create(t *testing.T) {
 				CreatedAt: createdAt,
 				UpdatedAt: updatedAt,
 			},
-			false,
+			true,
 		},
 		{
 			"empty basename",
@@ -376,14 +376,14 @@ func Test_fileStore_Update(t *testing.T) {
 				CreatedAt: createdAt,
 				UpdatedAt: updatedAt,
 			},
-			false,
+			true,
 		},
 		{
 			"clear zip",
 			&file.BaseFile{
 				ID:             fileIDs[fileIdxInZip],
-				Path:           getFilePath(folderIdxWithFiles, getFileBaseName(fileIdxZip)),
+				Path:           getFilePath(folderIdxWithFiles, getFileBaseName(fileIdxZip)+".renamed"),
-				Basename:       getFileBaseName(fileIdxZip),
+				Basename:       getFileBaseName(fileIdxZip) + ".renamed",
 				ParentFolderID: folderIDs[folderIdxWithFiles],
 			},
 			false,

pkg/sqlite/filter.go

@@ -420,9 +420,12 @@ func stringCriterionHandler(c *models.StringCriterionInput, column string) crite
 	}
 }
 
-func pathCriterionHandler(c *models.StringCriterionInput, pathColumn string, basenameColumn string) criterionHandlerFunc {
+func pathCriterionHandler(c *models.StringCriterionInput, pathColumn string, basenameColumn string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
 	return func(ctx context.Context, f *filterBuilder) {
 		if c != nil {
+			if addJoinFn != nil {
+				addJoinFn(f)
+			}
 			addWildcards := true
 			not := false
 

pkg/sqlite/gallery.go

@@ -374,10 +374,8 @@ func (qb *GalleryStore) findBySubquery(ctx context.Context, sq *goqu.SelectDatas
 }
 
 func (qb *GalleryStore) FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Gallery, error) {
-	table := qb.queryTable()
+	sq := dialect.From(galleriesFilesJoinTable).Select(galleriesFilesJoinTable.Col(galleryIDColumn)).Where(
-
+		galleriesFilesJoinTable.Col(fileIDColumn).Eq(fileID),
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
-		table.Col("file_id").Eq(fileID),
 	)
 
 	ret, err := qb.findBySubquery(ctx, sq)
@@ -396,18 +394,23 @@ func (qb *GalleryStore) CountByFileID(ctx context.Context, fileID file.ID) (int,
 }
 
 func (qb *GalleryStore) FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Gallery, error) {
-	table := qb.queryTable()
+	fingerprintTable := fingerprintTableMgr.table
 
 	var ex []exp.Expression
 
 	for _, v := range fp {
 		ex = append(ex, goqu.And(
-			table.Col("fingerprint_type").Eq(v.Type),
+			fingerprintTable.Col("type").Eq(v.Type),
-			table.Col("fingerprint").Eq(v.Fingerprint),
+			fingerprintTable.Col("fingerprint").Eq(v.Fingerprint),
 		))
 	}
 
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(goqu.Or(ex...))
+	sq := dialect.From(galleriesFilesJoinTable).
+		InnerJoin(
+			fingerprintTable,
+			goqu.On(fingerprintTable.Col(fileIDColumn).Eq(galleriesFilesJoinTable.Col(fileIDColumn))),
+		).
+		Select(galleriesFilesJoinTable.Col(galleryIDColumn)).Where(goqu.Or(ex...))
 
 	ret, err := qb.findBySubquery(ctx, sq)
 	if err != nil {
@@ -418,50 +421,55 @@ func (qb *GalleryStore) FindByFingerprints(ctx context.Context, fp []file.Finger
 }
 
 func (qb *GalleryStore) FindByChecksum(ctx context.Context, checksum string) ([]*models.Gallery, error) {
-	table := galleriesQueryTable
+	return qb.FindByFingerprints(ctx, []file.Fingerprint{
-
+		{
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
+			Type:        file.FingerprintTypeMD5,
-		table.Col("fingerprint_type").Eq(file.FingerprintTypeMD5),
+			Fingerprint: checksum,
-		table.Col("fingerprint").Eq(checksum),
+		},
-	)
+	})
-
-	ret, err := qb.findBySubquery(ctx, sq)
-	if err != nil {
-		return nil, fmt.Errorf("getting gallery by checksum %s: %w", checksum, err)
-	}
-
-	return ret, nil
 }
 
 func (qb *GalleryStore) FindByChecksums(ctx context.Context, checksums []string) ([]*models.Gallery, error) {
-	table := galleriesQueryTable
+	fingerprints := make([]file.Fingerprint, len(checksums))
 
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
+	for i, c := range checksums {
-		table.Col("fingerprint_type").Eq(file.FingerprintTypeMD5),
+		fingerprints[i] = file.Fingerprint{
-		table.Col("fingerprint").In(checksums),
+			Type:        file.FingerprintTypeMD5,
-	)
+			Fingerprint: c,
-
+		}
-	ret, err := qb.findBySubquery(ctx, sq)
-	if err != nil {
-		return nil, fmt.Errorf("getting gallery by checksums: %w", err)
 	}
-
+	return qb.FindByFingerprints(ctx, fingerprints)
-	return ret, nil
 }
 
 func (qb *GalleryStore) FindByPath(ctx context.Context, p string) ([]*models.Gallery, error) {
-	table := galleriesQueryTable
+	table := qb.table()
+	filesTable := fileTableMgr.table
+	fileFoldersTable := folderTableMgr.table.As("file_folders")
+	foldersTable := folderTableMgr.table
+
 	basename := filepath.Base(p)
 	dir, _ := path(filepath.Dir(p)).Value()
 	pp, _ := path(p).Value()
 
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
+	sq := dialect.From(table).LeftJoin(
+		galleriesFilesJoinTable,
+		goqu.On(galleriesFilesJoinTable.Col(galleryIDColumn).Eq(table.Col(idColumn))),
+	).InnerJoin(
+		filesTable,
+		goqu.On(filesTable.Col(idColumn).Eq(galleriesFilesJoinTable.Col(fileIDColumn))),
+	).LeftJoin(
+		fileFoldersTable,
+		goqu.On(fileFoldersTable.Col(idColumn).Eq(filesTable.Col("parent_folder_id"))),
+	).LeftJoin(
+		foldersTable,
+		goqu.On(foldersTable.Col(idColumn).Eq(table.Col("folder_id"))),
+	).Select(table.Col(idColumn)).Where(
 		goqu.Or(
 			goqu.And(
-				table.Col("parent_folder_path").Eq(dir),
+				fileFoldersTable.Col("path").Eq(dir),
-				table.Col("basename").Eq(basename),
+				filesTable.Col("basename").Eq(basename),
 			),
-			table.Col("folder_path").Eq(pp),
+			foldersTable.Col("path").Eq(pp),
 		),
 	)
 
@@ -474,7 +482,7 @@ func (qb *GalleryStore) FindByPath(ctx context.Context, p string) ([]*models.Gal
 }
 
 func (qb *GalleryStore) FindByFolderID(ctx context.Context, folderID file.FolderID) ([]*models.Gallery, error) {
-	table := galleriesQueryTable
+	table := qb.table()
 
 	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
 		table.Col("folder_id").Eq(folderID),
@@ -489,10 +497,8 @@ func (qb *GalleryStore) FindByFolderID(ctx context.Context, folderID file.Folder
 }
 
 func (qb *GalleryStore) FindBySceneID(ctx context.Context, sceneID int) ([]*models.Gallery, error) {
-	table := galleriesQueryTable
+	sq := dialect.From(galleriesScenesJoinTable).Select(galleriesScenesJoinTable.Col(galleryIDColumn)).Where(
-
+		galleriesScenesJoinTable.Col(sceneIDColumn).Eq(sceneID),
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
-		table.Col("scene_id").Eq(sceneID),
 	)
 
 	ret, err := qb.findBySubquery(ctx, sq)
@@ -504,13 +510,8 @@ func (qb *GalleryStore) FindBySceneID(ctx context.Context, sceneID int) ([]*mode
 }
 
 func (qb *GalleryStore) FindByImageID(ctx context.Context, imageID int) ([]*models.Gallery, error) {
-	table := galleriesQueryTable
+	sq := dialect.From(galleriesImagesJoinTable).Select(galleriesImagesJoinTable.Col(galleryIDColumn)).Where(
-
+		galleriesImagesJoinTable.Col(imageIDColumn).Eq(imageID),
-	sq := dialect.From(table).Select(table.Col(idColumn)).InnerJoin(
-		galleriesImagesJoinTable,
-		goqu.On(table.Col(idColumn).Eq(galleriesImagesJoinTable.Col(galleryIDColumn))),
-	).Where(
-		galleriesImagesJoinTable.Col("image_id").Eq(imageID),
 	)
 
 	ret, err := qb.findBySubquery(ctx, sq)
@@ -602,7 +603,7 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga
 		}
 	}))
 
-	query.handleCriterion(ctx, pathCriterionHandler(galleryFilter.Path, "galleries_query.parent_folder_path", "galleries_query.basename"))
+	query.handleCriterion(ctx, pathCriterionHandler(galleryFilter.Path, "galleries_query.parent_folder_path", "galleries_query.basename", nil))
 	query.handleCriterion(ctx, galleryFileCountCriterionHandler(qb, galleryFilter.FileCount))
 	query.handleCriterion(ctx, intCriterionHandler(galleryFilter.Rating, "galleries.rating"))
 	query.handleCriterion(ctx, stringCriterionHandler(galleryFilter.URL, "galleries.url"))

pkg/sqlite/gallery_test.go

@@ -1182,6 +1182,108 @@ func Test_galleryQueryBuilder_CountByImageID(t *testing.T) {
 	}
 }
 
+func galleriesToIDs(i []*models.Gallery) []int {
+	var ret []int
+	for _, ii := range i {
+		ret = append(ret, ii.ID)
+	}
+
+	return ret
+}
+
+func Test_galleryStore_FindByFileID(t *testing.T) {
+	tests := []struct {
+		name    string
+		fileID  file.ID
+		include []int
+		exclude []int
+	}{
+		{
+			"valid",
+			galleryFileIDs[galleryIdx1WithImage],
+			[]int{galleryIdx1WithImage},
+			nil,
+		},
+		{
+			"invalid",
+			invalidFileID,
+			nil,
+			[]int{galleryIdx1WithImage},
+		},
+	}
+
+	qb := db.Gallery
+
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			assert := assert.New(t)
+			got, err := qb.FindByFileID(ctx, tt.fileID)
+			if err != nil {
+				t.Errorf("GalleryStore.FindByFileID() error = %v", err)
+				return
+			}
+			for _, f := range got {
+				clearGalleryFileIDs(f)
+			}
+
+			ids := galleriesToIDs(got)
+			include := indexesToIDs(galleryIDs, tt.include)
+			exclude := indexesToIDs(galleryIDs, tt.exclude)
+
+			for _, i := range include {
+				assert.Contains(ids, i)
+			}
+			for _, e := range exclude {
+				assert.NotContains(ids, e)
+			}
+		})
+	}
+}
+
+func Test_galleryStore_FindByFolderID(t *testing.T) {
+	tests := []struct {
+		name     string
+		folderID file.FolderID
+		include  []int
+		exclude  []int
+	}{
+		// TODO - add folder gallery
+		{
+			"invalid",
+			invalidFolderID,
+			nil,
+			[]int{galleryIdxWithImage},
+		},
+	}
+
+	qb := db.Gallery
+
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			assert := assert.New(t)
+			got, err := qb.FindByFolderID(ctx, tt.folderID)
+			if err != nil {
+				t.Errorf("GalleryStore.FindByFolderID() error = %v", err)
+				return
+			}
+			for _, f := range got {
+				clearGalleryFileIDs(f)
+			}
+
+			ids := galleriesToIDs(got)
+			include := indexesToIDs(imageIDs, tt.include)
+			exclude := indexesToIDs(imageIDs, tt.exclude)
+
+			for _, i := range include {
+				assert.Contains(ids, i)
+			}
+			for _, e := range exclude {
+				assert.NotContains(ids, e)
+			}
+		})
+	}
+}
+
 func TestGalleryQueryQ(t *testing.T) {
 	withTxn(func(ctx context.Context) error {
 		const galleryIdx = 0

pkg/sqlite/image.go

@@ -3,9 +3,7 @@ package sqlite
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
-	"path/filepath"
 	"time"
 
 	"github.com/jmoiron/sqlx"
@@ -365,11 +363,14 @@ func (qb *ImageStore) findBySubquery(ctx context.Context, sq *goqu.SelectDataset
 }
 
 func (qb *ImageStore) FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Image, error) {
-	table := imagesQueryTable
+	table := qb.table()
 
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
+	sq := dialect.From(table).
-		table.Col("file_id").Eq(fileID),
+		InnerJoin(
-	)
+			imagesFilesJoinTable,
+			goqu.On(table.Col(idColumn).Eq(imagesFilesJoinTable.Col(imageIDColumn))),
+		).
+		Select(table.Col(idColumn)).Where(imagesFilesJoinTable.Col(fileIDColumn).Eq(fileID))
 
 	ret, err := qb.findBySubquery(ctx, sq)
 	if err != nil {
@@ -387,18 +388,28 @@ func (qb *ImageStore) CountByFileID(ctx context.Context, fileID file.ID) (int, e
 }
 
 func (qb *ImageStore) FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Image, error) {
-	table := imagesQueryTable
+	table := qb.table()
+	fingerprintTable := fingerprintTableMgr.table
 
 	var ex []exp.Expression
 
 	for _, v := range fp {
 		ex = append(ex, goqu.And(
-			table.Col("fingerprint_type").Eq(v.Type),
+			fingerprintTable.Col("type").Eq(v.Type),
-			table.Col("fingerprint").Eq(v.Fingerprint),
+			fingerprintTable.Col("fingerprint").Eq(v.Fingerprint),
 		))
 	}
 
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(goqu.Or(ex...))
+	sq := dialect.From(table).
+		InnerJoin(
+			imagesFilesJoinTable,
+			goqu.On(table.Col(idColumn).Eq(imagesFilesJoinTable.Col(imageIDColumn))),
+		).
+		InnerJoin(
+			fingerprintTable,
+			goqu.On(fingerprintTable.Col(fileIDColumn).Eq(imagesFilesJoinTable.Col(fileIDColumn))),
+		).
+		Select(table.Col(idColumn)).Where(goqu.Or(ex...))
 
 	ret, err := qb.findBySubquery(ctx, sq)
 	if err != nil {
@@ -409,45 +420,32 @@ func (qb *ImageStore) FindByFingerprints(ctx context.Context, fp []file.Fingerpr
 }
 
 func (qb *ImageStore) FindByChecksum(ctx context.Context, checksum string) ([]*models.Image, error) {
-	table := imagesQueryTable
+	return qb.FindByFingerprints(ctx, []file.Fingerprint{
-
+		{
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
+			Type:        file.FingerprintTypeMD5,
-		table.Col("fingerprint_type").Eq(file.FingerprintTypeMD5),
+			Fingerprint: checksum,
-		table.Col("fingerprint").Eq(checksum),
+		},
-	)
+	})
-
-	ret, err := qb.findBySubquery(ctx, sq)
-	if err != nil {
-		return nil, fmt.Errorf("getting image by checksum %s: %w", checksum, err)
-	}
-
-	return ret, nil
-}
-
-func (qb *ImageStore) FindByPath(ctx context.Context, p string) ([]*models.Image, error) {
-	table := imagesQueryTable
-	basename := filepath.Base(p)
-	dir, _ := path(filepath.Dir(p)).Value()
-
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
-		table.Col("parent_folder_path").Eq(dir),
-		table.Col("basename").Eq(basename),
-	)
-
-	ret, err := qb.findBySubquery(ctx, sq)
-	if err != nil && !errors.Is(err, sql.ErrNoRows) {
-		return nil, fmt.Errorf("getting image by path %s: %w", p, err)
-	}
-
-	return ret, nil
 }
 
 func (qb *ImageStore) FindByGalleryID(ctx context.Context, galleryID int) ([]*models.Image, error) {
-	table := qb.queryTable()
+	table := qb.table()
+	queryTable := qb.queryTable()
 
-	q := qb.selectDataset().Where(
+	sq := dialect.From(table).
-		table.Col("gallery_id").Eq(galleryID),
+		InnerJoin(
-	).GroupBy(table.Col(idColumn)).Order(table.Col("parent_folder_path").Asc(), table.Col("basename").Asc())
+			galleriesImagesJoinTable,
+			goqu.On(table.Col(idColumn).Eq(galleriesImagesJoinTable.Col(imageIDColumn))),
+		).
+		Select(table.Col(idColumn)).Where(
+		galleriesImagesJoinTable.Col("gallery_id").Eq(galleryID),
+	)
+
+	q := qb.selectDataset().Prepared(true).Where(
+		queryTable.Col(idColumn).Eq(
+			sq,
+		),
+	).Order(queryTable.Col("parent_folder_path").Asc(), queryTable.Col("basename").Asc())
 
 	ret, err := qb.getMany(ctx, q)
 	if err != nil {
@@ -465,8 +463,21 @@ func (qb *ImageStore) CountByGalleryID(ctx context.Context, galleryID int) (int,
 }
 
 func (qb *ImageStore) FindByFolderID(ctx context.Context, folderID file.FolderID) ([]*models.Image, error) {
-	table := qb.queryTable()
+	table := qb.table()
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(table.Col("parent_folder_id").Eq(folderID))
+	fileTable := goqu.T(fileTable)
+
+	sq := dialect.From(table).
+		InnerJoin(
+			imagesFilesJoinTable,
+			goqu.On(table.Col(idColumn).Eq(imagesFilesJoinTable.Col(imageIDColumn))),
+		).
+		InnerJoin(
+			fileTable,
+			goqu.On(imagesFilesJoinTable.Col(fileIDColumn).Eq(fileTable.Col(idColumn))),
+		).
+		Select(table.Col(idColumn)).Where(
+		fileTable.Col("parent_folder_id").Eq(folderID),
+	)
 
 	ret, err := qb.findBySubquery(ctx, sq)
 	if err != nil {
@@ -477,8 +488,21 @@ func (qb *ImageStore) FindByFolderID(ctx context.Context, folderID file.FolderID
 }
 
 func (qb *ImageStore) FindByZipFileID(ctx context.Context, zipFileID file.ID) ([]*models.Image, error) {
-	table := qb.queryTable()
+	table := qb.table()
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(table.Col("zip_file_id").Eq(zipFileID))
+	fileTable := goqu.T(fileTable)
+
+	sq := dialect.From(table).
+		InnerJoin(
+			imagesFilesJoinTable,
+			goqu.On(table.Col(idColumn).Eq(imagesFilesJoinTable.Col(imageIDColumn))),
+		).
+		InnerJoin(
+			fileTable,
+			goqu.On(imagesFilesJoinTable.Col(fileIDColumn).Eq(fileTable.Col(idColumn))),
+		).
+		Select(table.Col(idColumn)).Where(
+		fileTable.Col("zip_file_id").Eq(zipFileID),
+	)
 
 	ret, err := qb.findBySubquery(ctx, sq)
 	if err != nil {
@@ -563,20 +587,21 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF
 
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if imageFilter.Checksum != nil {
-			f.addLeftJoin(fingerprintTable, "fingerprints_md5", "galleries_query.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
+			qb.addQueryTable(f)
+			f.addInnerJoin(fingerprintTable, "fingerprints_md5", "galleries_query.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
 		}
 
 		stringCriterionHandler(imageFilter.Checksum, "fingerprints_md5.fingerprint")(ctx, f)
 	}))
 	query.handleCriterion(ctx, stringCriterionHandler(imageFilter.Title, "images.title"))
 
-	query.handleCriterion(ctx, pathCriterionHandler(imageFilter.Path, "images_query.parent_folder_path", "images_query.basename"))
+	query.handleCriterion(ctx, pathCriterionHandler(imageFilter.Path, "images_query.parent_folder_path", "images_query.basename", qb.addQueryTable))
 	query.handleCriterion(ctx, imageFileCountCriterionHandler(qb, imageFilter.FileCount))
 	query.handleCriterion(ctx, intCriterionHandler(imageFilter.Rating, "images.rating"))
 	query.handleCriterion(ctx, intCriterionHandler(imageFilter.OCounter, "images.o_counter"))
 	query.handleCriterion(ctx, boolCriterionHandler(imageFilter.Organized, "images.organized"))
 
-	query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "images_query.image_height", "images_query.image_width"))
+	query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "images_query.image_height", "images_query.image_width", qb.addQueryTable))
 	query.handleCriterion(ctx, imageIsMissingCriterionHandler(qb, imageFilter.IsMissing))
 
 	query.handleCriterion(ctx, imageTagsCriterionHandler(qb, imageFilter.Tags))
@@ -591,6 +616,10 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF
 	return query
 }
 
+func (qb *ImageStore) addQueryTable(f *filterBuilder) {
+	f.addInnerJoin(imagesQueryTable.GetTable(), "", "images.id = images_query.id")
+}
+
 func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) (*queryBuilder, error) {
 	if imageFilter == nil {
 		imageFilter = &models.ImageFilterType{}
@@ -602,15 +631,27 @@ func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFi
 	query := qb.newQuery()
 	distinctIDs(&query, imageTable)
 
-	// for convenience, join with the query view
-	query.addJoins(join{
-		table:    imagesQueryTable.GetTable(),
-		onClause: "images.id = images_query.id",
-		joinType: "INNER",
-	})
-
 	if q := findFilter.Q; q != nil && *q != "" {
-		searchColumns := []string{"images.title", "images_query.parent_folder_path", "images_query.basename", "images_query.fingerprint"}
+		query.addJoins(
+			join{
+				table:    imagesFilesTable,
+				onClause: "images_files.image_id = images.id",
+			},
+			join{
+				table:    fileTable,
+				onClause: "images_files.file_id = files.id",
+			},
+			join{
+				table:    folderTable,
+				onClause: "files.parent_folder_id = folders.id",
+			},
+			join{
+				table:    fingerprintTable,
+				onClause: "files_fingerprints.file_id = images_files.file_id",
+			},
+		)
+
+		searchColumns := []string{"images.title", "folders.path", "files.basename", "files_fingerprints.fingerprint"}
 		query.parseQueryString(searchColumns, *q)
 	}
 
@@ -621,7 +662,7 @@ func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFi
 
 	query.addFilter(filter)
 
-	query.sortAndPagination = qb.getImageSort(findFilter) + getPagination(findFilter)
+	qb.setImageSortAndPagination(&query, findFilter)
 
 	return &query, nil
 }
@@ -769,8 +810,10 @@ func imageTagCountCriterionHandler(qb *ImageStore, tagCount *models.IntCriterion
 
 func imageGalleriesCriterionHandler(qb *ImageStore, galleries *models.MultiCriterionInput) criterionHandlerFunc {
 	addJoinsFunc := func(f *filterBuilder) {
-		qb.galleriesRepository().join(f, "", "images.id")
+		if galleries.Modifier == models.CriterionModifierIncludes || galleries.Modifier == models.CriterionModifierIncludesAll {
-		f.addLeftJoin(galleryTable, "", "galleries_images.gallery_id = galleries.id")
+			f.addInnerJoin(galleriesImagesTable, "", "galleries_images.image_id = images.id")
+			f.addInnerJoin(galleryTable, "", "galleries_images.gallery_id = galleries.id")
+		}
 	}
 	h := qb.getMultiCriterionHandlerBuilder(galleryTable, galleriesImagesTable, galleryIDColumn, addJoinsFunc)
 
@@ -872,30 +915,54 @@ INNER JOIN (` + valuesClause + `) t ON t.column2 = pt.tag_id
 	}
 }
 
-func (qb *ImageStore) getImageSort(findFilter *models.FindFilterType) string {
+func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *models.FindFilterType) {
-	if findFilter == nil || findFilter.Sort == nil || *findFilter.Sort == "" {
+	sortClause := ""
-		return ""
-	}
-	sort := findFilter.GetSort("title")
-	direction := findFilter.GetDirection()
 
-	// translate sort field
+	if findFilter != nil && findFilter.Sort != nil && *findFilter.Sort != "" {
-	if sort == "file_mod_time" {
+		sort := findFilter.GetSort("title")
-		sort = "mod_time"
+		direction := findFilter.GetDirection()
+
+		// translate sort field
+		if sort == "file_mod_time" {
+			sort = "mod_time"
+		}
+
+		addFilesJoin := func() {
+			q.addJoins(
+				join{
+					table:    imagesFilesTable,
+					onClause: "images_files.image_id = images.id",
+				},
+				join{
+					table:    fileTable,
+					onClause: "images_files.file_id = files.id",
+				},
+			)
+		}
+
+		switch sort {
+		case "path":
+			addFilesJoin()
+			q.addJoins(join{
+				table:    folderTable,
+				onClause: "files.parent_folder_id = folders.id",
+			})
+			sortClause = " ORDER BY folders.path " + direction + ", files.basename " + direction
+		case "file_count":
+			sortClause = getCountSort(imageTable, imagesFilesTable, imageIDColumn, direction)
+		case "tag_count":
+			sortClause = getCountSort(imageTable, imagesTagsTable, imageIDColumn, direction)
+		case "performer_count":
+			sortClause = getCountSort(imageTable, performersImagesTable, imageIDColumn, direction)
+		case "mod_time", "size":
+			addFilesJoin()
+			sortClause = getSort(sort, direction, "files")
+		default:
+			sortClause = getSort(sort, direction, "images")
+		}
 	}
 
-	switch sort {
+	q.sortAndPagination = sortClause + getPagination(findFilter)
-	case "path":
-		return " ORDER BY images_query.parent_folder_path " + direction + ", images_query.basename " + direction
-	case "file_count":
-		return getCountSort(imageTable, imagesFilesTable, imageIDColumn, direction)
-	case "tag_count":
-		return getCountSort(imageTable, imagesTagsTable, imageIDColumn, direction)
-	case "performer_count":
-		return getCountSort(imageTable, performersImagesTable, imageIDColumn, direction)
-	default:
-		return getSort(sort, direction, "images_query")
-	}
 }
 
 func (qb *ImageStore) galleriesRepository() *joinRepository {

pkg/sqlite/image_test.go

@@ -1035,38 +1035,58 @@ func Test_imageQueryBuilder_FindByChecksum(t *testing.T) {
 	}
 }
 
-func Test_imageQueryBuilder_FindByPath(t *testing.T) {
+func Test_imageQueryBuilder_FindByFingerprints(t *testing.T) {
-	getPath := func(index int) string {
+	getChecksum := func(index int) string {
-		return getFilePath(folderIdxWithImageFiles, getImageBasename(index))
+		return getImageStringValue(index, checksumField)
 	}
 
 	tests := []struct {
-		name    string
+		name         string
-		path    string
+		fingerprints []file.Fingerprint
-		want    []*models.Image
+		want         []*models.Image
-		wantErr bool
+		wantErr      bool
 	}{
 		{
 			"valid",
-			getPath(imageIdxWithGallery),
+			[]file.Fingerprint{
+				{
+					Type:        file.FingerprintTypeMD5,
+					Fingerprint: getChecksum(imageIdxWithGallery),
+				},
+			},
 			[]*models.Image{makeImageWithID(imageIdxWithGallery)},
 			false,
 		},
 		{
 			"invalid",
-			"invalid path",
+			[]file.Fingerprint{
+				{
+					Type:        file.FingerprintTypeMD5,
+					Fingerprint: "invalid checksum",
+				},
+			},
 			nil,
 			false,
 		},
 		{
 			"with performers",
-			getPath(imageIdxWithTwoPerformers),
+			[]file.Fingerprint{
+				{
+					Type:        file.FingerprintTypeMD5,
+					Fingerprint: getChecksum(imageIdxWithTwoPerformers),
+				},
+			},
 			[]*models.Image{makeImageWithID(imageIdxWithTwoPerformers)},
 			false,
 		},
 		{
 			"with tags",
-			getPath(imageIdxWithTwoTags),
+			[]file.Fingerprint{
+				{
+					Type:        file.FingerprintTypeMD5,
+					Fingerprint: getChecksum(imageIdxWithTwoTags),
+				},
+			},
 			[]*models.Image{makeImageWithID(imageIdxWithTwoTags)},
 			false,
 		},
@@ -1077,14 +1097,16 @@ func Test_imageQueryBuilder_FindByPath(t *testing.T) {
 	for _, tt := range tests {
 		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
 			assert := assert.New(t)
-			got, err := qb.FindByPath(ctx, tt.path)
+			got, err := qb.FindByFingerprints(ctx, tt.fingerprints)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("imageQueryBuilder.FindByPath() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("imageQueryBuilder.FindByChecksum() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
+
 			for _, f := range got {
 				clearImageFileIDs(f)
 			}
+
 			assert.Equal(tt.want, got)
 		})
 	}
@@ -1178,6 +1200,55 @@ func imagesToIDs(i []*models.Image) []int {
 	return ret
 }
 
+func Test_imageStore_FindByFileID(t *testing.T) {
+	tests := []struct {
+		name    string
+		fileID  file.ID
+		include []int
+		exclude []int
+	}{
+		{
+			"valid",
+			imageFileIDs[imageIdxWithGallery],
+			[]int{imageIdxWithGallery},
+			nil,
+		},
+		{
+			"invalid",
+			invalidFileID,
+			nil,
+			[]int{imageIdxWithGallery},
+		},
+	}
+
+	qb := db.Image
+
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			assert := assert.New(t)
+			got, err := qb.FindByFileID(ctx, tt.fileID)
+			if err != nil {
+				t.Errorf("ImageStore.FindByFileID() error = %v", err)
+				return
+			}
+			for _, f := range got {
+				clearImageFileIDs(f)
+			}
+
+			ids := imagesToIDs(got)
+			include := indexesToIDs(imageIDs, tt.include)
+			exclude := indexesToIDs(imageIDs, tt.exclude)
+
+			for _, i := range include {
+				assert.Contains(ids, i)
+			}
+			for _, e := range exclude {
+				assert.NotContains(ids, e)
+			}
+		})
+	}
+}
+
 func Test_imageStore_FindByFolderID(t *testing.T) {
 	tests := []struct {
 		name     string

pkg/sqlite/migrations/32_files.up.sql

@@ -10,6 +10,7 @@ CREATE TABLE `folders` (
 );
 
 CREATE INDEX `index_folders_on_parent_folder_id` on `folders` (`parent_folder_id`);
+CREATE UNIQUE INDEX `index_folders_on_path_unique` on `folders` (`path`);
 
 -- require reference folders/zip files to be deleted manually first
 CREATE TABLE `files` (
@@ -26,12 +27,12 @@ CREATE TABLE `files` (
   CHECK (`basename` != '')
 );
 
-CREATE UNIQUE INDEX `index_files_zip_basename_unique` ON `files` (`zip_file_id`, `parent_folder_id`, `basename`);
+CREATE UNIQUE INDEX `index_files_zip_basename_unique` ON `files` (`zip_file_id`, `parent_folder_id`, `basename`) WHERE `zip_file_id` IS NOT NULL;
-CREATE INDEX `index_files_on_parent_folder_id_basename` on `files` (`parent_folder_id`, `basename`);
+CREATE UNIQUE INDEX `index_files_on_parent_folder_id_basename_unique` on `files` (`parent_folder_id`, `basename`);
 CREATE INDEX `index_files_on_basename` on `files` (`basename`);
 
 ALTER TABLE `folders` ADD COLUMN `zip_file_id` integer REFERENCES `files`(`id`);
-CREATE UNIQUE INDEX `index_folders_path_unique` on `folders` (`zip_file_id`, `path`);
+CREATE INDEX `index_folders_on_zip_file_id` on `folders` (`zip_file_id`) WHERE `zip_file_id` IS NOT NULL;
 
 CREATE TABLE `files_fingerprints` (
   `file_id` integer NOT NULL,
@@ -84,7 +85,7 @@ CREATE TABLE `images_files` (
     PRIMARY KEY(`image_id`, `file_id`)
 );
 
-CREATE INDEX `index_images_files_file_id` ON `images_files` (`file_id`);
+CREATE INDEX `index_images_files_on_file_id` on `images_files` (`file_id`);
 
 CREATE TABLE `galleries_files` (
     `gallery_id` integer NOT NULL,

pkg/sqlite/scene.go

@@ -456,10 +456,8 @@ func (qb *SceneStore) find(ctx context.Context, id int) (*models.Scene, error) {
 }
 
 func (qb *SceneStore) FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Scene, error) {
-	table := qb.queryTable()
+	sq := dialect.From(scenesFilesJoinTable).Select(scenesFilesJoinTable.Col(sceneIDColumn)).Where(
-
+		scenesFilesJoinTable.Col(fileIDColumn).Eq(fileID),
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
-		table.Col("file_id").Eq(fileID),
 	)
 
 	ret, err := qb.findBySubquery(ctx, sq)
@@ -478,18 +476,23 @@ func (qb *SceneStore) CountByFileID(ctx context.Context, fileID file.ID) (int, e
 }
 
 func (qb *SceneStore) FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Scene, error) {
-	table := qb.queryTable()
+	fingerprintTable := fingerprintTableMgr.table
 
 	var ex []exp.Expression
 
 	for _, v := range fp {
 		ex = append(ex, goqu.And(
-			table.Col("fingerprint_type").Eq(v.Type),
+			fingerprintTable.Col("type").Eq(v.Type),
-			table.Col("fingerprint").Eq(v.Fingerprint),
+			fingerprintTable.Col("fingerprint").Eq(v.Fingerprint),
 		))
 	}
 
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(goqu.Or(ex...))
+	sq := dialect.From(scenesFilesJoinTable).
+		InnerJoin(
+			fingerprintTable,
+			goqu.On(fingerprintTable.Col(fileIDColumn).Eq(scenesFilesJoinTable.Col(fileIDColumn))),
+		).
+		Select(scenesFilesJoinTable.Col(sceneIDColumn)).Where(goqu.Or(ex...))
 
 	ret, err := qb.findBySubquery(ctx, sq)
 	if err != nil {
@@ -500,39 +503,26 @@ func (qb *SceneStore) FindByFingerprints(ctx context.Context, fp []file.Fingerpr
 }
 
 func (qb *SceneStore) FindByChecksum(ctx context.Context, checksum string) ([]*models.Scene, error) {
-	table := qb.queryTable()
+	return qb.FindByFingerprints(ctx, []file.Fingerprint{
-
+		{
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
+			Type:        file.FingerprintTypeMD5,
-		table.Col("fingerprint_type").Eq(file.FingerprintTypeMD5),
+			Fingerprint: checksum,
-		table.Col("fingerprint").Eq(checksum),
+		},
-	)
+	})
-
-	ret, err := qb.findBySubquery(ctx, sq)
-	if err != nil {
-		return nil, fmt.Errorf("getting scenes by checksum %s: %w", checksum, err)
-	}
-
-	return ret, nil
 }
 
 func (qb *SceneStore) FindByOSHash(ctx context.Context, oshash string) ([]*models.Scene, error) {
-	table := qb.queryTable()
+	return qb.FindByFingerprints(ctx, []file.Fingerprint{
-
+		{
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
+			Type:        file.FingerprintTypeOshash,
-		table.Col("fingerprint_type").Eq(file.FingerprintTypeOshash),
+			Fingerprint: oshash,
-		table.Col("fingerprint").Eq(oshash),
+		},
-	)
+	})
-
-	ret, err := qb.findBySubquery(ctx, sq)
-	if err != nil {
-		return nil, fmt.Errorf("getting scenes by oshash %s: %w", oshash, err)
-	}
-
-	return ret, nil
 }
 
 func (qb *SceneStore) FindByPath(ctx context.Context, p string) ([]*models.Scene, error) {
-	table := scenesQueryTable
+	filesTable := fileTableMgr.table
+	foldersTable := folderTableMgr.table
 	basename := filepath.Base(p)
 	dirStr := filepath.Dir(p)
 
@@ -542,9 +532,15 @@ func (qb *SceneStore) FindByPath(ctx context.Context, p string) ([]*models.Scene
 
 	dir, _ := path(dirStr).Value()
 
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
+	sq := dialect.From(scenesFilesJoinTable).InnerJoin(
-		table.Col("parent_folder_path").Like(dir),
+		filesTable,
-		table.Col("basename").Like(basename),
+		goqu.On(filesTable.Col(idColumn).Eq(scenesFilesJoinTable.Col(fileIDColumn))),
+	).InnerJoin(
+		foldersTable,
+		goqu.On(foldersTable.Col(idColumn).Eq(filesTable.Col("parent_folder_id"))),
+	).Select(scenesFilesJoinTable.Col(sceneIDColumn)).Where(
+		foldersTable.Col("path").Like(dir),
+		filesTable.Col("basename").Like(basename),
 	)
 
 	ret, err := qb.findBySubquery(ctx, sq)
@@ -670,18 +666,16 @@ func (qb *SceneStore) CountByTagID(ctx context.Context, tagID int) (int, error)
 }
 
 func (qb *SceneStore) countMissingFingerprints(ctx context.Context, fpType string) (int, error) {
-	table := qb.queryTable()
 	fpTable := fingerprintTableMgr.table.As("fingerprints_temp")
 
-	q := dialect.Select(goqu.COUNT(goqu.DISTINCT(table.Col(idColumn)))).From(table).LeftJoin(
+	q := dialect.From(scenesFilesJoinTable).LeftJoin(
 		fpTable,
 		goqu.On(
-			table.Col("file_id").Eq(fpTable.Col("file_id")),
+			scenesFilesJoinTable.Col(fileIDColumn).Eq(fpTable.Col(fileIDColumn)),
 			fpTable.Col("type").Eq(fpType),
 		),
-	)
+	).Select(goqu.COUNT(goqu.DISTINCT(scenesFilesJoinTable.Col(sceneIDColumn)))).Where(fpTable.Col("fingerprint").IsNull())
 
-	q.Where(fpTable.Col("fingerprint").IsNull())
 	return count(ctx, q)
 }
 
@@ -762,7 +756,7 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
 		query.not(qb.makeFilter(ctx, sceneFilter.Not))
 	}
 
-	query.handleCriterion(ctx, pathCriterionHandler(sceneFilter.Path, "scenes_query.parent_folder_path", "scenes_query.basename"))
+	query.handleCriterion(ctx, pathCriterionHandler(sceneFilter.Path, "scenes_query.parent_folder_path", "scenes_query.basename", nil))
 	query.handleCriterion(ctx, sceneFileCountCriterionHandler(qb, sceneFilter.FileCount))
 	query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.Title, "scenes.title"))
 	query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.Details, "scenes.details"))
@@ -799,7 +793,7 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
 	query.handleCriterion(ctx, boolCriterionHandler(sceneFilter.Organized, "scenes.organized"))
 
 	query.handleCriterion(ctx, durationCriterionHandler(sceneFilter.Duration, "scenes_query.duration"))
-	query.handleCriterion(ctx, resolutionCriterionHandler(sceneFilter.Resolution, "scenes_query.video_height", "scenes_query.video_width"))
+	query.handleCriterion(ctx, resolutionCriterionHandler(sceneFilter.Resolution, "scenes_query.video_height", "scenes_query.video_width", nil))
 
 	query.handleCriterion(ctx, hasMarkersCriterionHandler(sceneFilter.HasMarkers))
 	query.handleCriterion(ctx, sceneIsMissingCriterionHandler(qb, sceneFilter.IsMissing))
@@ -958,9 +952,13 @@ func durationCriterionHandler(durationFilter *models.IntCriterionInput, column s
 	}
 }
 
-func resolutionCriterionHandler(resolution *models.ResolutionCriterionInput, heightColumn string, widthColumn string) criterionHandlerFunc {
+func resolutionCriterionHandler(resolution *models.ResolutionCriterionInput, heightColumn string, widthColumn string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
 	return func(ctx context.Context, f *filterBuilder) {
 		if resolution != nil && resolution.Value.IsValid() {
+			if addJoinFn != nil {
+				addJoinFn(f)
+			}
+
 			min := resolution.Value.GetMinResolution()
 			max := resolution.Value.GetMaxResolution()
 

pkg/sqlite/scene_test.go

@@ -1655,6 +1655,152 @@ func TestSceneCountByPerformerID(t *testing.T) {
 	})
 }
 
+func scenesToIDs(i []*models.Scene) []int {
+	var ret []int
+	for _, ii := range i {
+		ret = append(ret, ii.ID)
+	}
+
+	return ret
+}
+
+func Test_sceneStore_FindByFileID(t *testing.T) {
+	tests := []struct {
+		name    string
+		fileID  file.ID
+		include []int
+		exclude []int
+	}{
+		{
+			"valid",
+			sceneFileIDs[sceneIdx1WithPerformer],
+			[]int{sceneIdx1WithPerformer},
+			nil,
+		},
+		{
+			"invalid",
+			invalidFileID,
+			nil,
+			[]int{sceneIdx1WithPerformer},
+		},
+	}
+
+	qb := db.Scene
+
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			assert := assert.New(t)
+			got, err := qb.FindByFileID(ctx, tt.fileID)
+			if err != nil {
+				t.Errorf("SceneStore.FindByFileID() error = %v", err)
+				return
+			}
+			for _, f := range got {
+				clearSceneFileIDs(f)
+			}
+
+			ids := scenesToIDs(got)
+			include := indexesToIDs(galleryIDs, tt.include)
+			exclude := indexesToIDs(galleryIDs, tt.exclude)
+
+			for _, i := range include {
+				assert.Contains(ids, i)
+			}
+			for _, e := range exclude {
+				assert.NotContains(ids, e)
+			}
+		})
+	}
+}
+
+func Test_sceneStore_CountByFileID(t *testing.T) {
+	tests := []struct {
+		name   string
+		fileID file.ID
+		want   int
+	}{
+		{
+			"valid",
+			sceneFileIDs[sceneIdxWithTwoPerformers],
+			1,
+		},
+		{
+			"invalid",
+			invalidFileID,
+			0,
+		},
+	}
+
+	qb := db.Scene
+
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			assert := assert.New(t)
+			got, err := qb.CountByFileID(ctx, tt.fileID)
+			if err != nil {
+				t.Errorf("SceneStore.CountByFileID() error = %v", err)
+				return
+			}
+
+			assert.Equal(tt.want, got)
+		})
+	}
+}
+
+func Test_sceneStore_CountMissingChecksum(t *testing.T) {
+	tests := []struct {
+		name string
+		want int
+	}{
+		{
+			"valid",
+			0,
+		},
+	}
+
+	qb := db.Scene
+
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			assert := assert.New(t)
+			got, err := qb.CountMissingChecksum(ctx)
+			if err != nil {
+				t.Errorf("SceneStore.CountMissingChecksum() error = %v", err)
+				return
+			}
+
+			assert.Equal(tt.want, got)
+		})
+	}
+}
+
+func Test_sceneStore_CountMissingOshash(t *testing.T) {
+	tests := []struct {
+		name string
+		want int
+	}{
+		{
+			"valid",
+			0,
+		},
+	}
+
+	qb := db.Scene
+
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			assert := assert.New(t)
+			got, err := qb.CountMissingOSHash(ctx)
+			if err != nil {
+				t.Errorf("SceneStore.CountMissingOSHash() error = %v", err)
+				return
+			}
+
+			assert.Equal(tt.want, got)
+		})
+	}
+}
+
 func TestSceneWall(t *testing.T) {
 	withTxn(func(ctx context.Context) error {
 		sqb := db.Scene

scripts/test_db_generator/makeTestDB.go

@@ -1,5 +1,5 @@
-//go:build ignore
+//go:build tools
-// +build ignore
+// +build tools
 
 package main
 
@@ -14,12 +14,12 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/stashapp/stash/pkg/database"
+	"github.com/stashapp/stash/pkg/file"
 	"github.com/stashapp/stash/pkg/hash/md5"
-	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/sliceutil/intslice"
 	"github.com/stashapp/stash/pkg/sqlite"
+	"github.com/stashapp/stash/pkg/txn"
 	"gopkg.in/yaml.v2"
 )
 
@@ -40,8 +40,12 @@ type config struct {
 	Naming     namingConfig `yaml:"naming"`
 }
 
-var txnManager models.TransactionManager
+var (
-var c *config
+	repo     models.Repository
+	c        *config
+	db       *sqlite.Database
+	folderID file.FolderID
+)
 
 func main() {
 	rand.Seed(time.Now().UnixNano())
@@ -54,9 +58,17 @@ func main() {
 
 	initNaming(*c)
 
-	if err = database.Initialize(c.Database); err != nil {
+	db = sqlite.NewDatabase()
+	repo = db.TxnRepository()
+
+	logf("Initializing database...")
+	if err = db.Open(c.Database); err != nil {
 		log.Fatalf("couldn't initialize database: %v", err)
 	}
+	logf("Populating database...")
+	if err = makeFolder(); err != nil {
+		log.Fatalf("couldn't create folder: %v", err)
+	}
 	populateDB()
 }
 
@@ -89,12 +101,8 @@ func populateDB() {
 	makeMarkers(c.Markers)
 }
 
-func withTxn(f func(r models.Repository) error) error {
+func withTxn(f func(ctx context.Context) error) error {
-	if txnManager == nil {
+	return txn.WithTxn(context.Background(), db, f)
-		txnManager = sqlite.NewTransactionManager()
-	}
-
-	return txnManager.WithTxn(context.TODO(), f)
 }
 
 func retry(attempts int, fn func() error) error {
@@ -109,28 +117,43 @@ func retry(attempts int, fn func() error) error {
 	return err
 }
 
+func makeFolder() error {
+	return withTxn(func(ctx context.Context) error {
+		f := file.Folder{
+			Path: ".",
+		}
+		if err := repo.Folder.Create(ctx, &f); err != nil {
+			return err
+		}
+
+		folderID = f.ID
+		return nil
+	})
+}
+
 func makeTags(n int) {
+	logf("creating %d tags...", n)
 	for i := 0; i < n; i++ {
 		if err := retry(100, func() error {
-			return withTxn(func(r models.Repository) error {
+			return withTxn(func(ctx context.Context) error {
 				name := names[c.Naming.Tags].generateName(1)
 				tag := models.Tag{
 					Name: name,
 				}
 
-				created, err := r.Tag().Create(tag)
+				created, err := repo.Tag.Create(ctx, tag)
 				if err != nil {
 					return err
 				}
 
 				if rand.Intn(100) > 5 {
-					t, _, err := r.Tag().Query(nil, getRandomFilter(1))
+					t, _, err := repo.Tag.Query(ctx, nil, getRandomFilter(1))
 					if err != nil {
 						return err
 					}
 
 					if len(t) > 0 && t[0].ID != created.ID {
-						if err := r.Tag().UpdateParentTags(created.ID, []int{t[0].ID}); err != nil {
+						if err := repo.Tag.UpdateParentTags(ctx, created.ID, []int{t[0].ID}); err != nil {
 							return err
 						}
 					}
@@ -145,9 +168,10 @@ func makeTags(n int) {
 }
 
 func makeStudios(n int) {
+	logf("creating %d studios...", n)
 	for i := 0; i < n; i++ {
 		if err := retry(100, func() error {
-			return withTxn(func(r models.Repository) error {
+			return withTxn(func(ctx context.Context) error {
 				name := names[c.Naming.Tags].generateName(rand.Intn(5) + 1)
 				studio := models.Studio{
 					Name:     sql.NullString{String: name, Valid: true},
@@ -155,7 +179,7 @@ func makeStudios(n int) {
 				}
 
 				if rand.Intn(100) > 5 {
-					ss, _, err := r.Studio().Query(nil, getRandomFilter(1))
+					ss, _, err := repo.Studio.Query(ctx, nil, getRandomFilter(1))
 					if err != nil {
 						return err
 					}
@@ -168,7 +192,7 @@ func makeStudios(n int) {
 					}
 				}
 
-				_, err := r.Studio().Create(studio)
+				_, err := repo.Studio.Create(ctx, studio)
 				return err
 			})
 		}); err != nil {
@@ -178,9 +202,10 @@ func makeStudios(n int) {
 }
 
 func makePerformers(n int) {
+	logf("creating %d performers...", n)
 	for i := 0; i < n; i++ {
 		if err := retry(100, func() error {
-			return withTxn(func(r models.Repository) error {
+			return withTxn(func(ctx context.Context) error {
 				name := generatePerformerName()
 				performer := models.Performer{
 					Name:     sql.NullString{String: name, Valid: true},
@@ -193,7 +218,7 @@ func makePerformers(n int) {
 
 				// TODO - set tags
 
-				_, err := r.Performer().Create(performer)
+				_, err := repo.Performer.Create(ctx, performer)
 				if err != nil {
 					err = fmt.Errorf("error creating performer with name: %s: %s", performer.Name.String, err.Error())
 				}
@@ -205,23 +230,72 @@ func makePerformers(n int) {
 	}
 }
 
+func generateBaseFile(path string) *file.BaseFile {
+	return &file.BaseFile{
+		Path:           path,
+		Basename:       path,
+		ParentFolderID: folderID,
+		Fingerprints: []file.Fingerprint{
+			file.Fingerprint{
+				Type:        "md5",
+				Fingerprint: md5.FromString(path),
+			},
+			file.Fingerprint{
+				Type:        "oshash",
+				Fingerprint: md5.FromString(path),
+			},
+		},
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
+	}
+}
+
+func generateVideoFile(path string) file.File {
+	w, h := getResolution()
+
+	return &file.VideoFile{
+		BaseFile: generateBaseFile(path),
+		Duration: rand.Float64() * 14400,
+		Height:   h,
+		Width:    w,
+	}
+}
+
+func makeVideoFile(ctx context.Context, path string) (file.File, error) {
+	f := generateVideoFile(path)
+
+	if err := repo.File.Create(ctx, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func logf(f string, args ...interface{}) {
+	log.Printf(f+"\n", args...)
+}
+
 func makeScenes(n int) {
-	logger.Infof("creating %d scenes...", n)
+	logf("creating %d scenes...", n)
 	for i := 0; i < n; {
 		// do in batches of 1000
 		batch := i + batchSize
 
-		if err := withTxn(func(r models.Repository) error {
+		if err := withTxn(func(ctx context.Context) error {
 			for ; i < batch && i < n; i++ {
 				scene := generateScene(i)
-				scene.StudioID = getRandomStudioID(r)
+				scene.StudioID = getRandomStudioID(ctx)
+				makeSceneRelationships(ctx, &scene)
 
-				created, err := r.Scene().Create(scene)
+				path := md5.FromString("scene/" + strconv.Itoa(i))
+				f, err := makeVideoFile(ctx, path)
 				if err != nil {
 					return err
 				}
 
-				makeSceneRelationships(r, created.ID)
+				if err := repo.Scene.Create(ctx, &scene, []file.ID{f.Base().ID}); err != nil {
+					return err
+				}
 			}
 
 			return nil
@@ -229,14 +303,14 @@ func makeScenes(n int) {
 			panic(err)
 		}
 
-		logger.Infof("... created %d scenes", i)
+		logf("... created %d scenes", i)
 	}
 }
 
-func getResolution() (int64, int64) {
+func getResolution() (int, int) {
 	res := models.AllResolutionEnum[rand.Intn(len(models.AllResolutionEnum))]
-	h := int64(res.GetMaxResolution())
+	h := res.GetMaxResolution()
-	var w int64
+	var w int
 	if h == 240 || h == 480 || rand.Intn(10) == 9 {
 		w = h * 4 / 3
 	} else {
@@ -250,54 +324,69 @@ func getResolution() (int64, int64) {
 	return w, h
 }
 
-func getDate() string {
+func getDate() time.Time {
 	s := rand.Int63n(time.Now().Unix())
 
-	d := time.Unix(s, 0)
+	return time.Unix(s, 0)
-	return d.Format("2006-01-02")
 }
 
 func generateScene(i int) models.Scene {
-	path := md5.FromString("scene/" + strconv.Itoa(i))
-	w, h := getResolution()
-
 	return models.Scene{
-		Path:     path,
+		Title: names[c.Naming.Scenes].generateName(rand.Intn(7) + 1),
-		Title:    sql.NullString{String: names[c.Naming.Scenes].generateName(rand.Intn(7) + 1), Valid: true},
+		Date: &models.Date{
-		Checksum: sql.NullString{String: md5.FromString(path), Valid: true},
+			Time: getDate(),
-		OSHash:   sql.NullString{String: md5.FromString(path), Valid: true},
-		Duration: sql.NullFloat64{
-			Float64: rand.Float64() * 14400,
-			Valid:   true,
-		},
-		Height: models.NullInt64(h),
-		Width:  models.NullInt64(w),
-		Date: models.SQLiteDate{
-			String: getDate(),
-			Valid:  true,
 		},
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
 	}
 }
 
+func generateImageFile(path string) file.File {
+	w, h := getResolution()
+
+	return &file.ImageFile{
+		BaseFile: generateBaseFile(path),
+		Height:   h,
+		Width:    w,
+	}
+}
+
+func makeImageFile(ctx context.Context, path string) (file.File, error) {
+	f := generateImageFile(path)
+
+	if err := repo.File.Create(ctx, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
 func makeImages(n int) {
-	logger.Infof("creating %d images...", n)
+	logf("creating %d images...", n)
 	for i := 0; i < n; {
 		// do in batches of 1000
 		batch := i + batchSize
-		if err := withTxn(func(r models.Repository) error {
+		if err := withTxn(func(ctx context.Context) error {
 			for ; i < batch && i < n; i++ {
 				image := generateImage(i)
-				image.StudioID = getRandomStudioID(r)
+				image.StudioID = getRandomStudioID(ctx)
+				makeImageRelationships(ctx, &image)
 
-				created, err := r.Image().Create(image)
+				path := md5.FromString("image/" + strconv.Itoa(i))
+				f, err := makeImageFile(ctx, path)
 				if err != nil {
 					return err
 				}
 
-				makeImageRelationships(r, created.ID)
+				if err := repo.Image.Create(ctx, &models.ImageCreateInput{
+					Image:   &image,
+					FileIDs: []file.ID{f.Base().ID},
+				}); err != nil {
+					return err
+				}
 			}
 
-			logger.Infof("... created %d images", i)
+			logf("... created %d images", i)
 
 			return nil
 		}); err != nil {
@@ -307,36 +396,37 @@ func makeImages(n int) {
 }
 
 func generateImage(i int) models.Image {
-	path := md5.FromString("image/" + strconv.Itoa(i))
-
-	w, h := getResolution()
-
 	return models.Image{
-		Title:    sql.NullString{String: names[c.Naming.Images].generateName(rand.Intn(7) + 1), Valid: true},
+		Title:     names[c.Naming.Images].generateName(rand.Intn(7) + 1),
-		Path:     path,
+		CreatedAt: time.Now(),
-		Checksum: md5.FromString(path),
+		UpdatedAt: time.Now(),
-		Height:   models.NullInt64(h),
-		Width:    models.NullInt64(w),
 	}
 }
 
 func makeGalleries(n int) {
-	logger.Infof("creating %d galleries...", n)
+	logf("creating %d galleries...", n)
 	for i := 0; i < n; {
 		// do in batches of 1000
 		batch := i + batchSize
 
-		if err := withTxn(func(r models.Repository) error {
+		if err := withTxn(func(ctx context.Context) error {
 			for ; i < batch && i < n; i++ {
 				gallery := generateGallery(i)
-				gallery.StudioID = getRandomStudioID(r)
+				gallery.StudioID = getRandomStudioID(ctx)
+				gallery.TagIDs = getRandomTags(ctx, 0, 15)
+				gallery.PerformerIDs = getRandomPerformers(ctx)
 
-				created, err := r.Gallery().Create(gallery)
+				path := md5.FromString("gallery/" + strconv.Itoa(i))
+				f, err := makeZipFile(ctx, path)
 				if err != nil {
 					return err
 				}
 
-				makeGalleryRelationships(r, created.ID)
+				if err := repo.Gallery.Create(ctx, &gallery, []file.ID{f.Base().ID}); err != nil {
+					return err
+				}
+
+				makeGalleryRelationships(ctx, &gallery)
 			}
 
 			return nil
@@ -344,49 +434,60 @@ func makeGalleries(n int) {
 			panic(err)
 		}
 
-		logger.Infof("... created %d galleries", i)
+		logf("... created %d galleries", i)
 	}
 }
 
-func generateGallery(i int) models.Gallery {
+func generateZipFile(path string) file.File {
-	path := md5.FromString("gallery/" + strconv.Itoa(i))
+	return generateBaseFile(path)
+}
 
+func makeZipFile(ctx context.Context, path string) (file.File, error) {
+	f := generateZipFile(path)
+
+	if err := repo.File.Create(ctx, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func generateGallery(i int) models.Gallery {
 	return models.Gallery{
-		Title:    sql.NullString{String: names[c.Naming.Galleries].generateName(rand.Intn(7) + 1), Valid: true},
+		Title: names[c.Naming.Galleries].generateName(rand.Intn(7) + 1),
-		Path:     sql.NullString{String: path, Valid: true},
+		Date: &models.Date{
-		Checksum: md5.FromString(path),
+			Time: getDate(),
-		Date: models.SQLiteDate{
-			String: getDate(),
-			Valid:  true,
 		},
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
 	}
 }
 
 func makeMarkers(n int) {
-	logger.Infof("creating %d markers...", n)
+	logf("creating %d markers...", n)
 	for i := 0; i < n; {
 		// do in batches of 1000
 		batch := i + batchSize
-		if err := withTxn(func(r models.Repository) error {
+		if err := withTxn(func(ctx context.Context) error {
 			for ; i < batch && i < n; i++ {
 				marker := generateMarker(i)
 				marker.SceneID = models.NullInt64(int64(getRandomScene()))
-				marker.PrimaryTagID = getRandomTags(r, 1, 1)[0]
+				marker.PrimaryTagID = getRandomTags(ctx, 1, 1)[0]
 
-				created, err := r.SceneMarker().Create(marker)
+				created, err := repo.SceneMarker.Create(ctx, marker)
 				if err != nil {
 					return err
 				}
 
-				tags := getRandomTags(r, 0, 5)
+				tags := getRandomTags(ctx, 0, 5)
 				// remove primary tag
 				tags = intslice.IntExclude(tags, []int{marker.PrimaryTagID})
-				if err := r.SceneMarker().UpdateTags(created.ID, tags); err != nil {
+				if err := repo.SceneMarker.UpdateTags(ctx, created.ID, tags); err != nil {
 					return err
 				}
 			}
 
-			logger.Infof("... created %d markers", i)
+			logf("... created %d markers", i)
 
 			return nil
 		}); err != nil {
@@ -410,9 +511,9 @@ func getRandomFilter(n int) *models.FindFilterType {
 	}
 }
 
-func getRandomStudioID(r models.Repository) sql.NullInt64 {
+func getRandomStudioID(ctx context.Context) *int {
 	if rand.Intn(10) == 0 {
-		return sql.NullInt64{}
+		return nil
 	}
 
 	// s, _, err := r.Studio().Query(nil, getRandomFilter(1))
@@ -420,82 +521,44 @@ func getRandomStudioID(r models.Repository) sql.NullInt64 {
 	// 	panic(err)
 	// }
 
-	return sql.NullInt64{
+	v := rand.Intn(c.Studios) + 1
-		Int64: int64(rand.Int63n(int64(c.Studios)) + 1),
+	return &v
-		Valid: true,
-	}
 }
 
-func makeSceneRelationships(r models.Repository, id int) {
+func makeSceneRelationships(ctx context.Context, s *models.Scene) {
 	// add tags
-	tagIDs := getRandomTags(r, 0, 15)
+	s.TagIDs = getRandomTags(ctx, 0, 15)
-	if len(tagIDs) > 0 {
-		if err := r.Scene().UpdateTags(id, tagIDs); err != nil {
-			panic(err)
-		}
-	}
 
 	// add performers
-	performerIDs := getRandomPerformers(r)
+	s.PerformerIDs = getRandomPerformers(ctx)
-	if len(tagIDs) > 0 {
-		if err := r.Scene().UpdatePerformers(id, performerIDs); err != nil {
-			panic(err)
-		}
-	}
 }
 
-func makeImageRelationships(r models.Repository, id int) {
+func makeImageRelationships(ctx context.Context, i *models.Image) {
 	// there are typically many more images. For performance reasons
 	// only a small proportion should have tags/performers
 
 	// add tags
 	if rand.Intn(100) == 0 {
-		tagIDs := getRandomTags(r, 1, 15)
+		i.TagIDs = getRandomTags(ctx, 1, 15)
-		if len(tagIDs) > 0 {
-			if err := r.Image().UpdateTags(id, tagIDs); err != nil {
-				panic(err)
-			}
-		}
 	}
 
 	// add performers
 	if rand.Intn(100) <= 1 {
-		performerIDs := getRandomPerformers(r)
+		i.PerformerIDs = getRandomPerformers(ctx)
-		if len(performerIDs) > 0 {
-			if err := r.Image().UpdatePerformers(id, performerIDs); err != nil {
-				panic(err)
-			}
-		}
 	}
 }
 
-func makeGalleryRelationships(r models.Repository, id int) {
+func makeGalleryRelationships(ctx context.Context, g *models.Gallery) {
-	// add tags
-	tagIDs := getRandomTags(r, 0, 15)
-	if len(tagIDs) > 0 {
-		if err := r.Gallery().UpdateTags(id, tagIDs); err != nil {
-			panic(err)
-		}
-	}
-
-	// add performers
-	performerIDs := getRandomPerformers(r)
-	if len(tagIDs) > 0 {
-		if err := r.Gallery().UpdatePerformers(id, performerIDs); err != nil {
-			panic(err)
-		}
-	}
-
 	// add images
-	imageIDs := getRandomImages(r)
+	imageIDs := getRandomImages(ctx)
-	if len(tagIDs) > 0 {
+	if len(imageIDs) > 0 {
-		if err := r.Gallery().UpdateImages(id, imageIDs); err != nil {
+		if err := repo.Gallery.UpdateImages(ctx, g.ID, imageIDs); err != nil {
 			panic(err)
 		}
 	}
 }
 
-func getRandomPerformers(r models.Repository) []int {
+func getRandomPerformers(ctx context.Context) []int {
 	n := rand.Intn(5)
 
 	var ret []int
@@ -521,7 +584,7 @@ func getRandomScene() int {
 	return rand.Intn(c.Scenes) + 1
 }
 
-func getRandomTags(r models.Repository, min, max int) []int {
+func getRandomTags(ctx context.Context, min, max int) []int {
 	var n int
 	if min == max {
 		n = min
@@ -548,7 +611,7 @@ func getRandomTags(r models.Repository, min, max int) []int {
 	return ret
 }
 
-func getRandomImages(r models.Repository) []int {
+func getRandomImages(ctx context.Context) []int {
 	n := rand.Intn(500)
 
 	var ret []int

scripts/test_db_generator/naming.go

@@ -1,4 +1,5 @@
-// +build ignore
+//go:build tools
+// +build tools
 
 package main
 

ui/v2.5/src/App.tsx

@@ -171,6 +171,10 @@ export const App: React.FC = () => {
   }
 
   function maybeRenderReleaseNotes() {
+    if (setupMatch) {
+      return;
+    }
+
     const lastNoteSeen = (config.data?.configuration.ui as IUIConfig)
       ?.lastNoteSeen;
     const notes = releaseNotes.filter((n) => {

ui/v2.5/src/docs/en/Changelog/v0170.md

@@ -9,7 +9,6 @@ Please report all issues to the following Github issue: https://github.com/stash
 ### 💥 Known issues
 * Import/export functionality is currently disabled. Needs further design.
 * Missing covers are not currently regenerated. Need to consider further, especially around scene cover redesign.
-* Deleting galleries is currently slow.
 
 ### ✨ New Features
 * Added support for identical files. Identical files are assigned to the same scene/gallery/image and can be viewed in File Info. ([#2676](https://github.com/stashapp/stash/pull/2676))

ui/v2.5/src/docs/en/ReleaseNotes/index.ts

@@ -10,7 +10,7 @@ interface IReleaseNotes {
 
 export const releaseNotes: IReleaseNotes[] = [
   {
-    date: 20220715,
+    date: 20220801,
     content: v0170,
   },
 ];

ui/v2.5/src/docs/en/ReleaseNotes/v0170.md

@@ -6,14 +6,13 @@ After migrating, please run a scan on your entire library to populate missing da
 
 Please report all issues to the following Github issue: https://github.com/stashapp/stash/issues/2737
 
+### **Warning:** if you are upgrading from an older `files-refactor` build, you will need to re-migrate your system from a schema version 31 database.
+
 ### 💥 Known issues
 * Import/export functionality is currently disabled. Needs further design.
 * Missing covers are not currently regenerated. Need to consider further, especially around scene cover redesign.
-* Deleting galleries is currently slow.
-
 
 ### Other changes:
-
 * Added support for filtering and sorting by file count. ([#2744](https://github.com/stashapp/stash/pull/2744))
 * Changelog has been moved from the stats page to a section in the Settings page.
 * Object titles are now displayed as the file basename if the title is not explicitly set. The `Don't include file extension as part of the title` scan flag is no longer supported.