[Files Refactor] Performance tuning (#2809)

* Use cache during migration
* Avoid use of query views
* Use FindMany to find related objects
* Log slow queries
* Add folders to generated files
* Use SlimScene for scene queries
* Include filename in migration error message

graphql/documents/queries/scene.graphql

@@ -4,7 +4,7 @@ query FindScenes($filter: FindFilterType, $scene_filter: SceneFilterType, $scene
     filesize
     duration
     scenes {
-      ...SceneData
+      ...SlimSceneData
     }
   }
 }

internal/api/resolver_model_gallery.go

@@ -147,7 +147,7 @@ func (r *galleryResolver) Date(ctx context.Context, obj *models.Gallery) (*strin
 func (r *galleryResolver) Scenes(ctx context.Context, obj *models.Gallery) (ret []*models.Scene, err error) {
 	if err := r.withTxn(ctx, func(ctx context.Context) error {
 		var err error
-		ret, err = r.repository.Scene.FindByGalleryID(ctx, obj.ID)
+		ret, err = r.repository.Scene.FindMany(ctx, obj.SceneIDs)
 		return err
 	}); err != nil {
 		return nil, err
@@ -175,7 +175,7 @@ func (r *galleryResolver) Studio(ctx context.Context, obj *models.Gallery) (ret
 func (r *galleryResolver) Tags(ctx context.Context, obj *models.Gallery) (ret []*models.Tag, err error) {
 	if err := r.withTxn(ctx, func(ctx context.Context) error {
 		var err error
-		ret, err = r.repository.Tag.FindByGalleryID(ctx, obj.ID)
+		ret, err = r.repository.Tag.FindMany(ctx, obj.TagIDs)
 		return err
 	}); err != nil {
 		return nil, err
@@ -187,7 +187,7 @@ func (r *galleryResolver) Tags(ctx context.Context, obj *models.Gallery) (ret []
 func (r *galleryResolver) Performers(ctx context.Context, obj *models.Gallery) (ret []*models.Performer, err error) {
 	if err := r.withTxn(ctx, func(ctx context.Context) error {
 		var err error
-		ret, err = r.repository.Performer.FindByGalleryID(ctx, obj.ID)
+		ret, err = r.repository.Performer.FindMany(ctx, obj.PerformerIDs)
 		return err
 	}); err != nil {
 		return nil, err

internal/api/resolver_model_scene.go

@@ -164,7 +164,7 @@ func (r *sceneResolver) Captions(ctx context.Context, obj *models.Scene) (ret []
 
 func (r *sceneResolver) Galleries(ctx context.Context, obj *models.Scene) (ret []*models.Gallery, err error) {
 	if err := r.withTxn(ctx, func(ctx context.Context) error {
-		ret, err = r.repository.Gallery.FindBySceneID(ctx, obj.ID)
+		ret, err = r.repository.Gallery.FindMany(ctx, obj.GalleryIDs)
 		return err
 	}); err != nil {
 		return nil, err
@@ -216,7 +216,7 @@ func (r *sceneResolver) Movies(ctx context.Context, obj *models.Scene) (ret []*S
 
 func (r *sceneResolver) Tags(ctx context.Context, obj *models.Scene) (ret []*models.Tag, err error) {
 	if err := r.withTxn(ctx, func(ctx context.Context) error {
-		ret, err = r.repository.Tag.FindBySceneID(ctx, obj.ID)
+		ret, err = r.repository.Tag.FindMany(ctx, obj.TagIDs)
 		return err
 	}); err != nil {
 		return nil, err
@@ -227,7 +227,7 @@ func (r *sceneResolver) Tags(ctx context.Context, obj *models.Scene) (ret []*mod
 
 func (r *sceneResolver) Performers(ctx context.Context, obj *models.Scene) (ret []*models.Performer, err error) {
 	if err := r.withTxn(ctx, func(ctx context.Context) error {
-		ret, err = r.repository.Performer.FindBySceneID(ctx, obj.ID)
+		ret, err = r.repository.Performer.FindMany(ctx, obj.PerformerIDs)
 		return err
 	}); err != nil {
 		return nil, err

pkg/sqlite/filter.go

@@ -517,7 +517,7 @@ func getPathSearchClause(pathColumn, basenameColumn, p string, addWildcards, not
 	return ret
 }
 
-func intCriterionHandler(c *models.IntCriterionInput, column string) criterionHandlerFunc {
+func intCriterionHandler(c *models.IntCriterionInput, column string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
 	return func(ctx context.Context, f *filterBuilder) {
 		if c != nil {
 			clause, args := getIntCriterionWhereClause(column, *c)
@@ -526,9 +526,12 @@ func intCriterionHandler(c *models.IntCriterionInput, column string) criterionHa
 	}
 }
 
-func boolCriterionHandler(c *bool, column string) criterionHandlerFunc {
+func boolCriterionHandler(c *bool, column string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
 	return func(ctx context.Context, f *filterBuilder) {
 		if c != nil {
+			if addJoinFn != nil {
+				addJoinFn(f)
+			}
 			var v string
 			if *c {
 				v = "1"

pkg/sqlite/gallery.go

@@ -587,7 +587,8 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga
 
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if galleryFilter.Checksum != nil {
-			f.addLeftJoin(fingerprintTable, "fingerprints_md5", "galleries_query.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
+			qb.addGalleriesFilesTable(f)
+			f.addLeftJoin(fingerprintTable, "fingerprints_md5", "galleries_files.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
 		}
 
 		stringCriterionHandler(galleryFilter.Checksum, "fingerprints_md5.fingerprint")(ctx, f)
@@ -595,19 +596,21 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga
 
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if galleryFilter.IsZip != nil {
+			qb.addGalleriesFilesTable(f)
 			if *galleryFilter.IsZip {
-				f.addWhere("galleries_query.file_id IS NOT NULL")
+
+				f.addWhere("galleries_files.file_id IS NOT NULL")
 			} else {
-				f.addWhere("galleries_query.file_id IS NULL")
+				f.addWhere("galleries_files.file_id IS NULL")
 			}
 		}
 	}))
 
-	query.handleCriterion(ctx, pathCriterionHandler(galleryFilter.Path, "galleries_query.parent_folder_path", "galleries_query.basename", nil))
+	query.handleCriterion(ctx, pathCriterionHandler(galleryFilter.Path, "folders.path", "files.basename", qb.addFoldersTable))
 	query.handleCriterion(ctx, galleryFileCountCriterionHandler(qb, galleryFilter.FileCount))
-	query.handleCriterion(ctx, intCriterionHandler(galleryFilter.Rating, "galleries.rating"))
+	query.handleCriterion(ctx, intCriterionHandler(galleryFilter.Rating, "galleries.rating", nil))
 	query.handleCriterion(ctx, stringCriterionHandler(galleryFilter.URL, "galleries.url"))
-	query.handleCriterion(ctx, boolCriterionHandler(galleryFilter.Organized, "galleries.organized"))
+	query.handleCriterion(ctx, boolCriterionHandler(galleryFilter.Organized, "galleries.organized", nil))
 	query.handleCriterion(ctx, galleryIsMissingCriterionHandler(qb, galleryFilter.IsMissing))
 	query.handleCriterion(ctx, galleryTagsCriterionHandler(qb, galleryFilter.Tags))
 	query.handleCriterion(ctx, galleryTagCountCriterionHandler(qb, galleryFilter.TagCount))
@@ -623,6 +626,20 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga
 	return query
 }
 
+func (qb *GalleryStore) addGalleriesFilesTable(f *filterBuilder) {
+	f.addLeftJoin(galleriesFilesTable, "", "galleries_files.gallery_id = galleries.id")
+}
+
+func (qb *GalleryStore) addFilesTable(f *filterBuilder) {
+	qb.addGalleriesFilesTable(f)
+	f.addLeftJoin(fileTable, "", "galleries_files.file_id = files.id")
+}
+
+func (qb *GalleryStore) addFoldersTable(f *filterBuilder) {
+	qb.addFilesTable(f)
+	f.addLeftJoin(folderTable, "", "files.parent_folder_id = folders.id")
+}
+
 func (qb *GalleryStore) makeQuery(ctx context.Context, galleryFilter *models.GalleryFilterType, findFilter *models.FindFilterType) (*queryBuilder, error) {
 	if galleryFilter == nil {
 		galleryFilter = &models.GalleryFilterType{}
@@ -634,16 +651,33 @@ func (qb *GalleryStore) makeQuery(ctx context.Context, galleryFilter *models.Gal
 	query := qb.newQuery()
 	distinctIDs(&query, galleryTable)
 
-	// for convenience, join with the query view
-	query.addJoins(join{
-		table:    galleriesQueryTable.GetTable(),
-		onClause: "galleries.id = galleries_query.id",
-		joinType: "INNER",
-	})
-
 	if q := findFilter.Q; q != nil && *q != "" {
+		query.addJoins(
+			join{
+				table:    galleriesFilesTable,
+				onClause: "galleries_files.gallery_id = galleries.id",
+			},
+			join{
+				table:    fileTable,
+				onClause: "galleries_files.file_id = files.id",
+			},
+			join{
+				table:    folderTable,
+				onClause: "files.parent_folder_id = folders.id",
+			},
+			join{
+				table:    fingerprintTable,
+				onClause: "files_fingerprints.file_id = galleries_files.file_id",
+			},
+			join{
+				table:    folderTable,
+				as:       "gallery_folder",
+				onClause: "galleries.folder_id = gallery_folder.id",
+			},
+		)
+
 		// add joins for files and checksum
-		searchColumns := []string{"galleries.title", "galleries_query.folder_path", "galleries_query.parent_folder_path", "galleries_query.basename", "galleries_query.fingerprint"}
+		searchColumns := []string{"galleries.title", "gallery_folder.path", "folders.path", "files.basename", "files_fingerprints.fingerprint"}
 		query.parseQueryString(searchColumns, *q)
 	}
 
@@ -654,7 +688,8 @@ func (qb *GalleryStore) makeQuery(ctx context.Context, galleryFilter *models.Gal
 
 	query.addFilter(filter)
 
-	query.sortAndPagination = qb.getGallerySort(findFilter) + getPagination(findFilter)
+	qb.setGallerySort(&query, findFilter)
+	query.sortAndPagination += getPagination(findFilter)
 
 	return &query, nil
 }
@@ -902,33 +937,52 @@ func galleryAverageResolutionCriterionHandler(qb *GalleryStore, resolution *mode
 	}
 }
 
-func (qb *GalleryStore) getGallerySort(findFilter *models.FindFilterType) string {
+func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.FindFilterType) {
 	if findFilter == nil || findFilter.Sort == nil || *findFilter.Sort == "" {
-		return ""
+		return
 	}
 
 	sort := findFilter.GetSort("path")
 	direction := findFilter.GetDirection()
 
-	// translate sort field
+	addFileTable := func() {
-	if sort == "file_mod_time" {
+		query.addJoins(
-		sort = "mod_time"
+			join{
+				table:    galleriesFilesTable,
+				onClause: "galleries_files.gallery_id = galleries.id",
+			},
+			join{
+				table:    fileTable,
+				onClause: "galleries_files.file_id = files.id",
+			},
+		)
 	}
 
 	switch sort {
 	case "file_count":
-		return getCountSort(galleryTable, galleriesFilesTable, galleryIDColumn, direction)
+		query.sortAndPagination += getCountSort(galleryTable, galleriesFilesTable, galleryIDColumn, direction)
 	case "images_count":
-		return getCountSort(galleryTable, galleriesImagesTable, galleryIDColumn, direction)
+		query.sortAndPagination += getCountSort(galleryTable, galleriesImagesTable, galleryIDColumn, direction)
 	case "tag_count":
-		return getCountSort(galleryTable, galleriesTagsTable, galleryIDColumn, direction)
+		query.sortAndPagination += getCountSort(galleryTable, galleriesTagsTable, galleryIDColumn, direction)
 	case "performer_count":
-		return getCountSort(galleryTable, performersGalleriesTable, galleryIDColumn, direction)
+		query.sortAndPagination += getCountSort(galleryTable, performersGalleriesTable, galleryIDColumn, direction)
 	case "path":
 		// special handling for path
-		return fmt.Sprintf(" ORDER BY galleries_query.parent_folder_path %s, galleries_query.basename %[1]s", direction)
+		addFileTable()
+		query.addJoins(
+			join{
+				table:    folderTable,
+				onClause: "files.parent_folder_id = folders.id",
+			},
+		)
+		query.sortAndPagination += fmt.Sprintf(" ORDER BY folders.path %s, files.basename %[1]s", direction)
+	case "file_mod_time":
+		sort = "mod_time"
+		addFileTable()
+		query.sortAndPagination += getSort(sort, direction, fileTable)
 	default:
-		return getSort(sort, direction, "galleries_query")
+		query.sortAndPagination += getSort(sort, direction, "galleries")
 	}
 }
 

pkg/sqlite/image.go

@@ -587,21 +587,21 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF
 
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if imageFilter.Checksum != nil {
-			qb.addQueryTable(f)
+			qb.addImagesFilesTable(f)
-			f.addInnerJoin(fingerprintTable, "fingerprints_md5", "galleries_query.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
+			f.addInnerJoin(fingerprintTable, "fingerprints_md5", "images_files.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
 		}
 
 		stringCriterionHandler(imageFilter.Checksum, "fingerprints_md5.fingerprint")(ctx, f)
 	}))
 	query.handleCriterion(ctx, stringCriterionHandler(imageFilter.Title, "images.title"))
 
-	query.handleCriterion(ctx, pathCriterionHandler(imageFilter.Path, "images_query.parent_folder_path", "images_query.basename", qb.addQueryTable))
+	query.handleCriterion(ctx, pathCriterionHandler(imageFilter.Path, "folders.path", "files.basename", qb.addFoldersTable))
 	query.handleCriterion(ctx, imageFileCountCriterionHandler(qb, imageFilter.FileCount))
-	query.handleCriterion(ctx, intCriterionHandler(imageFilter.Rating, "images.rating"))
+	query.handleCriterion(ctx, intCriterionHandler(imageFilter.Rating, "images.rating", nil))
-	query.handleCriterion(ctx, intCriterionHandler(imageFilter.OCounter, "images.o_counter"))
+	query.handleCriterion(ctx, intCriterionHandler(imageFilter.OCounter, "images.o_counter", nil))
-	query.handleCriterion(ctx, boolCriterionHandler(imageFilter.Organized, "images.organized"))
+	query.handleCriterion(ctx, boolCriterionHandler(imageFilter.Organized, "images.organized", nil))
 
-	query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "images_query.image_height", "images_query.image_width", qb.addQueryTable))
+	query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "image_files.height", "image_files.width", qb.addImageFilesTable))
 	query.handleCriterion(ctx, imageIsMissingCriterionHandler(qb, imageFilter.IsMissing))
 
 	query.handleCriterion(ctx, imageTagsCriterionHandler(qb, imageFilter.Tags))
@@ -616,8 +616,23 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF
 	return query
 }
 
-func (qb *ImageStore) addQueryTable(f *filterBuilder) {
+func (qb *ImageStore) addImagesFilesTable(f *filterBuilder) {
-	f.addInnerJoin(imagesQueryTable.GetTable(), "", "images.id = images_query.id")
+	f.addLeftJoin(imagesFilesTable, "", "images_files.image_id = images.id")
+}
+
+func (qb *ImageStore) addFilesTable(f *filterBuilder) {
+	qb.addImagesFilesTable(f)
+	f.addLeftJoin(fileTable, "", "images_files.file_id = files.id")
+}
+
+func (qb *ImageStore) addFoldersTable(f *filterBuilder) {
+	qb.addFilesTable(f)
+	f.addLeftJoin(folderTable, "", "files.parent_folder_id = folders.id")
+}
+
+func (qb *ImageStore) addImageFilesTable(f *filterBuilder) {
+	qb.addImagesFilesTable(f)
+	f.addLeftJoin(imageFileTable, "", "image_files.file_id = images_files.file_id")
 }
 
 func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) (*queryBuilder, error) {

pkg/sqlite/migrations/32_postmigrate.go

@@ -209,7 +209,7 @@ func (m *schema32Migrator) migrateFiles(ctx context.Context) error {
 
 					_, err = m.db.Exec("UPDATE `files` SET `parent_folder_id` = ?, `zip_file_id` = ?, `basename` = ? WHERE `id` = ?", parentID, zipFileID, basename, id)
 					if err != nil {
-						return err
+						return fmt.Errorf("migrating file %s: %w", p, err)
 					}
 				}
 
@@ -277,10 +277,23 @@ func (m *schema32Migrator) createFolderHierarchy(p string) (*int, sql.NullInt64,
 		return m.getOrCreateFolder(p, nil, sql.NullInt64{})
 	}
 
-	parentID, zipFileID, err := m.createFolderHierarchy(parent)
+	var (
+		parentID  *int
+		zipFileID sql.NullInt64
+		err       error
+	)
+
+	// try to find parent folder in cache first
+	foundEntry, ok := m.folderCache[parent]
+	if ok {
+		parentID = &foundEntry.id
+		zipFileID = foundEntry.zipID
+	} else {
+		parentID, zipFileID, err = m.createFolderHierarchy(parent)
 		if err != nil {
 			return nil, sql.NullInt64{}, err
 		}
+	}
 
 	return m.getOrCreateFolder(p, parentID, zipFileID)
 }
@@ -323,12 +336,12 @@ func (m *schema32Migrator) getOrCreateFolder(path string, parentID *int, zipFile
 	now := time.Now()
 	result, err := m.db.Exec(insertSQL, path, parentFolderID, zipFileID, time.Time{}, now, now)
 	if err != nil {
-		return nil, sql.NullInt64{}, err
+		return nil, sql.NullInt64{}, fmt.Errorf("creating folder %s: %w", path, err)
 	}
 
 	id, err := result.LastInsertId()
 	if err != nil {
-		return nil, sql.NullInt64{}, err
+		return nil, sql.NullInt64{}, fmt.Errorf("creating folder %s: %w", path, err)
 	}
 
 	idInt := int(id)

pkg/sqlite/movies.go

@@ -120,8 +120,8 @@ func (qb *movieQueryBuilder) makeFilter(ctx context.Context, movieFilter *models
 	query.handleCriterion(ctx, stringCriterionHandler(movieFilter.Name, "movies.name"))
 	query.handleCriterion(ctx, stringCriterionHandler(movieFilter.Director, "movies.director"))
 	query.handleCriterion(ctx, stringCriterionHandler(movieFilter.Synopsis, "movies.synopsis"))
-	query.handleCriterion(ctx, intCriterionHandler(movieFilter.Rating, "movies.rating"))
+	query.handleCriterion(ctx, intCriterionHandler(movieFilter.Rating, "movies.rating", nil))
-	query.handleCriterion(ctx, durationCriterionHandler(movieFilter.Duration, "movies.duration"))
+	query.handleCriterion(ctx, durationCriterionHandler(movieFilter.Duration, "movies.duration", nil))
 	query.handleCriterion(ctx, movieIsMissingCriterionHandler(qb, movieFilter.IsMissing))
 	query.handleCriterion(ctx, stringCriterionHandler(movieFilter.URL, "movies.url"))
 	query.handleCriterion(ctx, movieStudioCriterionHandler(qb, movieFilter.Studios))

pkg/sqlite/performer.go

@@ -245,8 +245,8 @@ func (qb *performerQueryBuilder) makeFilter(ctx context.Context, filter *models.
 	query.handleCriterion(ctx, stringCriterionHandler(filter.Name, tableName+".name"))
 	query.handleCriterion(ctx, stringCriterionHandler(filter.Details, tableName+".details"))
 
-	query.handleCriterion(ctx, boolCriterionHandler(filter.FilterFavorites, tableName+".favorite"))
+	query.handleCriterion(ctx, boolCriterionHandler(filter.FilterFavorites, tableName+".favorite", nil))
-	query.handleCriterion(ctx, boolCriterionHandler(filter.IgnoreAutoTag, tableName+".ignore_auto_tag"))
+	query.handleCriterion(ctx, boolCriterionHandler(filter.IgnoreAutoTag, tableName+".ignore_auto_tag", nil))
 
 	query.handleCriterion(ctx, yearFilterCriterionHandler(filter.BirthYear, tableName+".birthdate"))
 	query.handleCriterion(ctx, yearFilterCriterionHandler(filter.DeathYear, tableName+".death_date"))
@@ -269,10 +269,10 @@ func (qb *performerQueryBuilder) makeFilter(ctx context.Context, filter *models.
 	query.handleCriterion(ctx, stringCriterionHandler(filter.CareerLength, tableName+".career_length"))
 	query.handleCriterion(ctx, stringCriterionHandler(filter.Tattoos, tableName+".tattoos"))
 	query.handleCriterion(ctx, stringCriterionHandler(filter.Piercings, tableName+".piercings"))
-	query.handleCriterion(ctx, intCriterionHandler(filter.Rating, tableName+".rating"))
+	query.handleCriterion(ctx, intCriterionHandler(filter.Rating, tableName+".rating", nil))
 	query.handleCriterion(ctx, stringCriterionHandler(filter.HairColor, tableName+".hair_color"))
 	query.handleCriterion(ctx, stringCriterionHandler(filter.URL, tableName+".url"))
-	query.handleCriterion(ctx, intCriterionHandler(filter.Weight, tableName+".weight"))
+	query.handleCriterion(ctx, intCriterionHandler(filter.Weight, tableName+".weight", nil))
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if filter.StashID != nil {
 			qb.stashIDRepository().join(f, "performer_stash_ids", "performers.id")

pkg/sqlite/query.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 )
 
@@ -58,7 +57,6 @@ func (qb queryBuilder) toSQL(includeSortPagination bool) string {
 func (qb queryBuilder) findIDs(ctx context.Context) ([]int, error) {
 	const includeSortPagination = true
 	sql := qb.toSQL(includeSortPagination)
-	logger.Tracef("SQL: %s, args: %v", sql, qb.args)
 	return qb.repository.runIdsQuery(ctx, sql, qb.args)
 }
 

pkg/sqlite/repository.go

@@ -11,7 +11,6 @@ import (
 	"github.com/jmoiron/sqlx"
 
 	"github.com/stashapp/stash/pkg/file"
-	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 )
 
@@ -154,8 +153,6 @@ func (r *repository) runIdsQuery(ctx context.Context, query string, args []inter
 }
 
 func (r *repository) queryFunc(ctx context.Context, query string, args []interface{}, single bool, f func(rows *sqlx.Rows) error) error {
-	logger.Tracef("SQL: %s, args: %v", query, args)
-
 	rows, err := r.tx.Queryx(ctx, query, args...)
 
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {
@@ -252,8 +249,6 @@ func (r *repository) executeFindQuery(ctx context.Context, body string, args []i
 	idsQuery := withClause + body + sortAndPagination
 
 	// Perform query and fetch result
-	logger.Tracef("SQL: %s, args: %v", idsQuery, args)
-
 	var countResult int
 	var countErr error
 	var idsResult []int

pkg/sqlite/scene.go

@@ -756,13 +756,14 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
 		query.not(qb.makeFilter(ctx, sceneFilter.Not))
 	}
 
-	query.handleCriterion(ctx, pathCriterionHandler(sceneFilter.Path, "scenes_query.parent_folder_path", "scenes_query.basename", nil))
+	query.handleCriterion(ctx, pathCriterionHandler(sceneFilter.Path, "folders.path", "files.basename", qb.addFoldersTable))
 	query.handleCriterion(ctx, sceneFileCountCriterionHandler(qb, sceneFilter.FileCount))
 	query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.Title, "scenes.title"))
 	query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.Details, "scenes.details"))
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if sceneFilter.Oshash != nil {
-			f.addLeftJoin(fingerprintTable, "fingerprints_oshash", "scenes_query.file_id = fingerprints_oshash.file_id AND fingerprints_oshash.type = 'oshash'")
+			qb.addSceneFilesTable(f)
+			f.addLeftJoin(fingerprintTable, "fingerprints_oshash", "scenes_files.file_id = fingerprints_oshash.file_id AND fingerprints_oshash.type = 'oshash'")
 		}
 
 		stringCriterionHandler(sceneFilter.Oshash, "fingerprints_oshash.fingerprint")(ctx, f)
@@ -770,7 +771,8 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
 
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if sceneFilter.Checksum != nil {
-			f.addLeftJoin(fingerprintTable, "fingerprints_md5", "scenes_query.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
+			qb.addSceneFilesTable(f)
+			f.addLeftJoin(fingerprintTable, "fingerprints_md5", "scenes_files.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
 		}
 
 		stringCriterionHandler(sceneFilter.Checksum, "fingerprints_md5.fingerprint")(ctx, f)
@@ -778,22 +780,23 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
 
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if sceneFilter.Phash != nil {
-			f.addLeftJoin(fingerprintTable, "fingerprints_phash", "scenes_query.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'")
+			qb.addSceneFilesTable(f)
+			f.addLeftJoin(fingerprintTable, "fingerprints_phash", "scenes_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'")
 
 			value, _ := utils.StringToPhash(sceneFilter.Phash.Value)
 			intCriterionHandler(&models.IntCriterionInput{
 				Value:    int(value),
 				Modifier: sceneFilter.Phash.Modifier,
-			}, "fingerprints_phash.fingerprint")(ctx, f)
+			}, "fingerprints_phash.fingerprint", nil)(ctx, f)
 		}
 	}))
 
-	query.handleCriterion(ctx, intCriterionHandler(sceneFilter.Rating, "scenes.rating"))
+	query.handleCriterion(ctx, intCriterionHandler(sceneFilter.Rating, "scenes.rating", nil))
-	query.handleCriterion(ctx, intCriterionHandler(sceneFilter.OCounter, "scenes.o_counter"))
+	query.handleCriterion(ctx, intCriterionHandler(sceneFilter.OCounter, "scenes.o_counter", nil))
-	query.handleCriterion(ctx, boolCriterionHandler(sceneFilter.Organized, "scenes.organized"))
+	query.handleCriterion(ctx, boolCriterionHandler(sceneFilter.Organized, "scenes.organized", nil))
 
-	query.handleCriterion(ctx, durationCriterionHandler(sceneFilter.Duration, "scenes_query.duration"))
+	query.handleCriterion(ctx, durationCriterionHandler(sceneFilter.Duration, "video_files.duration", qb.addVideoFilesTable))
-	query.handleCriterion(ctx, resolutionCriterionHandler(sceneFilter.Resolution, "scenes_query.video_height", "scenes_query.video_width", nil))
+	query.handleCriterion(ctx, resolutionCriterionHandler(sceneFilter.Resolution, "video_files.height", "video_files.width", qb.addVideoFilesTable))
 
 	query.handleCriterion(ctx, hasMarkersCriterionHandler(sceneFilter.HasMarkers))
 	query.handleCriterion(ctx, sceneIsMissingCriterionHandler(qb, sceneFilter.IsMissing))
@@ -806,8 +809,8 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
 		}
 	}))
 
-	query.handleCriterion(ctx, boolCriterionHandler(sceneFilter.Interactive, "scenes.interactive"))
+	query.handleCriterion(ctx, boolCriterionHandler(sceneFilter.Interactive, "video_files.interactive", qb.addVideoFilesTable))
-	query.handleCriterion(ctx, intCriterionHandler(sceneFilter.InteractiveSpeed, "scenes.interactive_speed"))
+	query.handleCriterion(ctx, intCriterionHandler(sceneFilter.InteractiveSpeed, "video_files.interactive_speed", qb.addVideoFilesTable))
 
 	query.handleCriterion(ctx, sceneCaptionCriterionHandler(qb, sceneFilter.Captions))
 
@@ -820,11 +823,30 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
 	query.handleCriterion(ctx, scenePerformerTagsCriterionHandler(qb, sceneFilter.PerformerTags))
 	query.handleCriterion(ctx, scenePerformerFavoriteCriterionHandler(sceneFilter.PerformerFavorite))
 	query.handleCriterion(ctx, scenePerformerAgeCriterionHandler(sceneFilter.PerformerAge))
-	query.handleCriterion(ctx, scenePhashDuplicatedCriterionHandler(sceneFilter.Duplicated))
+	query.handleCriterion(ctx, scenePhashDuplicatedCriterionHandler(sceneFilter.Duplicated, qb.addSceneFilesTable))
 
 	return query
 }
 
+func (qb *SceneStore) addSceneFilesTable(f *filterBuilder) {
+	f.addLeftJoin(scenesFilesTable, "", "scenes_files.scene_id = scenes.id")
+}
+
+func (qb *SceneStore) addFilesTable(f *filterBuilder) {
+	qb.addSceneFilesTable(f)
+	f.addLeftJoin(fileTable, "", "scenes_files.file_id = files.id")
+}
+
+func (qb *SceneStore) addFoldersTable(f *filterBuilder) {
+	qb.addFilesTable(f)
+	f.addLeftJoin(folderTable, "", "files.parent_folder_id = folders.id")
+}
+
+func (qb *SceneStore) addVideoFilesTable(f *filterBuilder) {
+	qb.addSceneFilesTable(f)
+	f.addLeftJoin(videoFileTable, "", "video_files.file_id = scenes_files.file_id")
+}
+
 func (qb *SceneStore) Query(ctx context.Context, options models.SceneQueryOptions) (*models.SceneQueryResult, error) {
 	sceneFilter := options.SceneFilter
 	findFilter := options.FindFilter
@@ -839,17 +861,31 @@ func (qb *SceneStore) Query(ctx context.Context, options models.SceneQueryOption
 	query := qb.newQuery()
 	distinctIDs(&query, sceneTable)
 
-	// for convenience, join with the query view
-	query.addJoins(join{
-		table:    scenesQueryTable.GetTable(),
-		onClause: "scenes.id = scenes_query.id",
-		joinType: "INNER",
-	})
-
 	if q := findFilter.Q; q != nil && *q != "" {
-		query.join("scene_markers", "", "scene_markers.scene_id = scenes.id")
+		query.addJoins(
+			join{
+				table:    scenesFilesTable,
+				onClause: "scenes_files.scene_id = scenes.id",
+			},
+			join{
+				table:    fileTable,
+				onClause: "scenes_files.file_id = files.id",
+			},
+			join{
+				table:    folderTable,
+				onClause: "files.parent_folder_id = folders.id",
+			},
+			join{
+				table:    fingerprintTable,
+				onClause: "files_fingerprints.file_id = scenes_files.file_id",
+			},
+			join{
+				table:    sceneMarkerTable,
+				onClause: "scene_markers.scene_id = scenes.id",
+			},
+		)
 
-		searchColumns := []string{"scenes.title", "scenes.details", "scenes_query.parent_folder_path", "scenes_query.basename", "scenes_query.fingerprint", "scene_markers.title"}
+		searchColumns := []string{"scenes.title", "scenes.details", "folders.path", "files.basename", "files_fingerprints.fingerprint", "scene_markers.title"}
 		query.parseQueryString(searchColumns, *q)
 	}
 
@@ -890,12 +926,32 @@ func (qb *SceneStore) queryGroupedFields(ctx context.Context, options models.Sce
 	}
 
 	if options.TotalDuration {
-		query.addColumn("COALESCE(scenes_query.duration, 0) as duration")
+		query.addJoins(
+			join{
+				table:    scenesFilesTable,
+				onClause: "scenes_files.scene_id = scenes.id",
+			},
+			join{
+				table:    videoFileTable,
+				onClause: "scenes_files.file_id = video_files.file_id",
+			},
+		)
+		query.addColumn("COALESCE(video_files.duration, 0) as duration")
 		aggregateQuery.addColumn("SUM(temp.duration) as duration")
 	}
 
 	if options.TotalSize {
-		query.addColumn("COALESCE(scenes_query.size, 0) as size")
+		query.addJoins(
+			join{
+				table:    scenesFilesTable,
+				onClause: "scenes_files.scene_id = scenes.id",
+			},
+			join{
+				table:    fileTable,
+				onClause: "scenes_files.file_id = files.id",
+			},
+		)
+		query.addColumn("COALESCE(files.size, 0) as size")
 		aggregateQuery.addColumn("SUM(temp.size) as size")
 	}
 
@@ -928,24 +984,32 @@ func sceneFileCountCriterionHandler(qb *SceneStore, fileCount *models.IntCriteri
 	return h.handler(fileCount)
 }
 
-func scenePhashDuplicatedCriterionHandler(duplicatedFilter *models.PHashDuplicationCriterionInput) criterionHandlerFunc {
+func scenePhashDuplicatedCriterionHandler(duplicatedFilter *models.PHashDuplicationCriterionInput, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
 	return func(ctx context.Context, f *filterBuilder) {
 		// TODO: Wishlist item: Implement Distance matching
 		if duplicatedFilter != nil {
+			if addJoinFn != nil {
+				addJoinFn(f)
+			}
+
 			var v string
 			if *duplicatedFilter.Duplicated {
 				v = ">"
 			} else {
 				v = "="
 			}
-			f.addInnerJoin("(SELECT file_id FROM files_fingerprints INNER JOIN (SELECT fingerprint FROM files_fingerprints WHERE type = 'phash' GROUP BY fingerprint HAVING COUNT (fingerprint) "+v+" 1) dupes on files_fingerprints.fingerprint = dupes.fingerprint)", "scph", "scenes_query.file_id = scph.file_id")
+
+			f.addInnerJoin("(SELECT file_id FROM files_fingerprints INNER JOIN (SELECT fingerprint FROM files_fingerprints WHERE type = 'phash' GROUP BY fingerprint HAVING COUNT (fingerprint) "+v+" 1) dupes on files_fingerprints.fingerprint = dupes.fingerprint)", "scph", "scenes_files.file_id = scph.file_id")
 		}
 	}
 }
 
-func durationCriterionHandler(durationFilter *models.IntCriterionInput, column string) criterionHandlerFunc {
+func durationCriterionHandler(durationFilter *models.IntCriterionInput, column string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
 	return func(ctx context.Context, f *filterBuilder) {
 		if durationFilter != nil {
+			if addJoinFn != nil {
+				addJoinFn(f)
+			}
 			clause, args := getIntCriterionWhereClause("cast("+column+" as int)", *durationFilter)
 			f.addWhere(clause, args...)
 		}
@@ -1015,7 +1079,8 @@ func sceneIsMissingCriterionHandler(qb *SceneStore, isMissing *string) criterion
 				qb.stashIDRepository().join(f, "scene_stash_ids", "scenes.id")
 				f.addWhere("scene_stash_ids.scene_id IS NULL")
 			case "phash":
-				f.addLeftJoin(fingerprintTable, "fingerprints_phash", "scenes_query.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'")
+				qb.addSceneFilesTable(f)
+				f.addLeftJoin(fingerprintTable, "fingerprints_phash", "scenes_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'")
 				f.addWhere("fingerprints_phash.fingerprint IS NULL")
 			default:
 				f.addWhere("(scenes." + *isMissing + " IS NULL OR TRIM(scenes." + *isMissing + ") = '')")
@@ -1040,7 +1105,8 @@ func sceneCaptionCriterionHandler(qb *SceneStore, captions *models.StringCriteri
 		joinTable:    videoCaptionsTable,
 		stringColumn: captionCodeColumn,
 		addJoinTable: func(f *filterBuilder) {
-			f.addLeftJoin(videoCaptionsTable, "", "video_captions.file_id = scenes_query.file_id")
+			qb.addSceneFilesTable(f)
+			f.addLeftJoin(videoCaptionsTable, "", "video_captions.file_id = scenes_files.file_id")
 		},
 	}
 
@@ -1201,14 +1267,27 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF
 	}
 	sort := findFilter.GetSort("title")
 
-	// translate sort field
+	addFileTable := func() {
-	switch sort {
+		query.addJoins(
-	case "bitrate":
+			join{
-		sort = "bit_rate"
+				table:    scenesFilesTable,
-	case "file_mod_time":
+				onClause: "scenes_files.scene_id = scenes.id",
-		sort = "mod_time"
+			},
-	case "framerate":
+			join{
-		sort = "frame_rate"
+				table:    fileTable,
+				onClause: "scenes_files.file_id = files.id",
+			},
+		)
+	}
+
+	addVideoFileTable := func() {
+		addFileTable()
+		query.addJoins(
+			join{
+				table:    videoFileTable,
+				onClause: "video_files.file_id = scenes_files.file_id",
+			},
+		)
 	}
 
 	direction := findFilter.GetDirection()
@@ -1224,21 +1303,47 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF
 		query.sortAndPagination += getCountSort(sceneTable, scenesFilesTable, sceneIDColumn, direction)
 	case "path":
 		// special handling for path
-		query.sortAndPagination += fmt.Sprintf(" ORDER BY scenes_query.parent_folder_path %s, scenes_query.basename %[1]s", direction)
+		addFileTable()
+		query.addJoins(
+			join{
+				table:    folderTable,
+				onClause: "files.parent_folder_id = folders.id",
+			},
+		)
+		query.sortAndPagination += fmt.Sprintf(" ORDER BY folders.path %s, files.basename %[1]s", direction)
 	case "perceptual_similarity":
 		// special handling for phash
-		query.addJoins(join{
+		addFileTable()
+		query.addJoins(
+			join{
 				table:    fingerprintTable,
 				as:       "fingerprints_phash",
-			onClause: "scenes_query.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'",
+				onClause: "scenes_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'",
-		})
+			},
+		)
 
-		query.sortAndPagination += " ORDER BY fingerprints_phash.fingerprint " + direction + ", scenes_query.size DESC"
+		query.sortAndPagination += " ORDER BY fingerprints_phash.fingerprint " + direction + ", files.size DESC"
+	case "bitrate":
+		sort = "bit_rate"
+		addVideoFileTable()
+		query.sortAndPagination += getSort(sort, direction, videoFileTable)
+	case "file_mod_time":
+		sort = "mod_time"
+		addFileTable()
+		query.sortAndPagination += getSort(sort, direction, fileTable)
+	case "framerate":
+		sort = "frame_rate"
+		addVideoFileTable()
+		query.sortAndPagination += getSort(sort, direction, videoFileTable)
+	case "size":
+		addFileTable()
+		query.sortAndPagination += getSort(sort, direction, fileTable)
+	case "duration":
+		addVideoFileTable()
+		query.sortAndPagination += getSort(sort, direction, videoFileTable)
 	default:
-		query.sortAndPagination += getSort(sort, direction, "scenes_query")
+		query.sortAndPagination += getSort(sort, direction, "scenes")
 	}
-
-	query.sortAndPagination += ", scenes_query.bit_rate DESC, scenes_query.frame_rate DESC, scenes.rating DESC, scenes_query.duration DESC"
 }
 
 func (qb *SceneStore) imageRepository() *imageRepository {

pkg/sqlite/scene_test.go

@@ -1853,8 +1853,11 @@ func queryScene(ctx context.Context, t *testing.T, sqb models.SceneReader, scene
 	result, err := sqb.Query(ctx, models.SceneQueryOptions{
 		QueryOptions: models.QueryOptions{
 			FindFilter: findFilter,
+			Count:      true,
 		},
 		SceneFilter:   sceneFilter,
+		TotalDuration: true,
+		TotalSize:     true,
 	})
 	if err != nil {
 		t.Errorf("Error querying scene: %v", err)
@@ -1875,7 +1878,9 @@ func sceneQueryQ(ctx context.Context, t *testing.T, sqb models.SceneReader, q st
 	}
 	scenes := queryScene(ctx, t, sqb, nil, &filter)
 
-	assert.Len(t, scenes, 1)
+	if !assert.Len(t, scenes, 1) {
+		return
+	}
 	scene := scenes[0]
 	assert.Equal(t, sceneIDs[expectedSceneIdx], scene.ID)
 

pkg/sqlite/studio.go

@@ -207,8 +207,8 @@ func (qb *studioQueryBuilder) makeFilter(ctx context.Context, studioFilter *mode
 	query.handleCriterion(ctx, stringCriterionHandler(studioFilter.Name, studioTable+".name"))
 	query.handleCriterion(ctx, stringCriterionHandler(studioFilter.Details, studioTable+".details"))
 	query.handleCriterion(ctx, stringCriterionHandler(studioFilter.URL, studioTable+".url"))
-	query.handleCriterion(ctx, intCriterionHandler(studioFilter.Rating, studioTable+".rating"))
+	query.handleCriterion(ctx, intCriterionHandler(studioFilter.Rating, studioTable+".rating", nil))
-	query.handleCriterion(ctx, boolCriterionHandler(studioFilter.IgnoreAutoTag, studioTable+".ignore_auto_tag"))
+	query.handleCriterion(ctx, boolCriterionHandler(studioFilter.IgnoreAutoTag, studioTable+".ignore_auto_tag", nil))
 
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if studioFilter.StashID != nil {

pkg/sqlite/table.go

@@ -557,7 +557,6 @@ func queryFunc(ctx context.Context, query *goqu.SelectDataset, single bool, f fu
 		return err
 	}
 
-	logger.Tracef("SQL: %s [%v]", q, args)
 	rows, err := tx.QueryxContext(ctx, q, args...)
 
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {
@@ -592,7 +591,6 @@ func querySimple(ctx context.Context, query *goqu.SelectDataset, out interface{}
 		return err
 	}
 
-	logger.Tracef("SQL: %s [%v]", q, args)
 	rows, err := tx.QueryxContext(ctx, q, args...)
 	if err != nil {
 		return fmt.Errorf("querying `%s` [%v]: %w", q, args, err)

pkg/sqlite/tag.go

@@ -297,7 +297,7 @@ func (qb *tagQueryBuilder) makeFilter(ctx context.Context, tagFilter *models.Tag
 
 	query.handleCriterion(ctx, stringCriterionHandler(tagFilter.Name, tagTable+".name"))
 	query.handleCriterion(ctx, tagAliasCriterionHandler(qb, tagFilter.Aliases))
-	query.handleCriterion(ctx, boolCriterionHandler(tagFilter.IgnoreAutoTag, tagTable+".ignore_auto_tag"))
+	query.handleCriterion(ctx, boolCriterionHandler(tagFilter.IgnoreAutoTag, tagTable+".ignore_auto_tag", nil))
 
 	query.handleCriterion(ctx, tagIsMissingCriterionHandler(qb, tagFilter.IsMissing))
 	query.handleCriterion(ctx, tagSceneCountCriterionHandler(qb, tagFilter.SceneCount))

pkg/sqlite/tx.go

@@ -3,8 +3,14 @@ package sqlite
 import (
 	"context"
 	"database/sql"
+	"time"
 
 	"github.com/jmoiron/sqlx"
+	"github.com/stashapp/stash/pkg/logger"
+)
+
+const (
+	slowLogTime = time.Millisecond * 200
 )
 
 type dbReader interface {
@@ -14,6 +20,15 @@ type dbReader interface {
 	QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error)
 }
 
+func logSQL(start time.Time, query string, args ...interface{}) {
+	since := time.Since(start)
+	if since >= slowLogTime {
+		logger.Debugf("SLOW SQL [%v]: %s, args: %v", since, query, args)
+	} else {
+		logger.Tracef("SQL [%v]: %s, args: %v", since, query, args)
+	}
+}
+
 type dbWrapper struct{}
 
 func (*dbWrapper) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
@@ -22,7 +37,11 @@ func (*dbWrapper) Get(ctx context.Context, dest interface{}, query string, args
 		return err
 	}
 
-	return tx.Get(dest, query, args...)
+	start := time.Now()
+	err = tx.Get(dest, query, args...)
+	logSQL(start, query, args...)
+
+	return err
 }
 
 func (*dbWrapper) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
@@ -31,7 +50,11 @@ func (*dbWrapper) Select(ctx context.Context, dest interface{}, query string, ar
 		return err
 	}
 
-	return tx.Select(dest, query, args...)
+	start := time.Now()
+	err = tx.Select(dest, query, args...)
+	logSQL(start, query, args...)
+
+	return err
 }
 
 func (*dbWrapper) Queryx(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) {
@@ -40,7 +63,11 @@ func (*dbWrapper) Queryx(ctx context.Context, query string, args ...interface{})
 		return nil, err
 	}
 
-	return tx.Queryx(query, args...)
+	start := time.Now()
+	ret, err := tx.Queryx(query, args...)
+	logSQL(start, query, args...)
+
+	return ret, err
 }
 
 func (*dbWrapper) NamedExec(ctx context.Context, query string, arg interface{}) (sql.Result, error) {
@@ -49,7 +76,11 @@ func (*dbWrapper) NamedExec(ctx context.Context, query string, arg interface{})
 		return nil, err
 	}
 
-	return tx.NamedExec(query, arg)
+	start := time.Now()
+	ret, err := tx.NamedExec(query, arg)
+	logSQL(start, query, arg)
+
+	return ret, err
 }
 
 func (*dbWrapper) Exec(ctx context.Context, query string, args ...interface{}) (sql.Result, error) {
@@ -58,5 +89,9 @@ func (*dbWrapper) Exec(ctx context.Context, query string, args ...interface{}) (
 		return nil, err
 	}
 
-	return tx.Exec(query, args...)
+	start := time.Now()
+	ret, err := tx.Exec(query, args...)
+	logSQL(start, query, args...)
+
+	return ret, err
 }

scripts/test_db_generator/makeTestDB.go

@@ -11,10 +11,12 @@ import (
 	"math"
 	"math/rand"
 	"os"
+	"path"
 	"strconv"
 	"time"
 
 	"github.com/stashapp/stash/pkg/file"
+	"github.com/stashapp/stash/pkg/fsutil"
 	"github.com/stashapp/stash/pkg/hash/md5"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/sliceutil/intslice"
@@ -66,9 +68,6 @@ func main() {
 		log.Fatalf("couldn't initialize database: %v", err)
 	}
 	logf("Populating database...")
-	if err = makeFolder(); err != nil {
-		log.Fatalf("couldn't create folder: %v", err)
-	}
 	populateDB()
 }
 
@@ -117,18 +116,39 @@ func retry(attempts int, fn func() error) error {
 	return err
 }
 
-func makeFolder() error {
+func getOrCreateFolder(ctx context.Context, p string) (*file.Folder, error) {
-	return withTxn(func(ctx context.Context) error {
+	ret, err := repo.Folder.FindByPath(ctx, p)
-		f := file.Folder{
+	if err != nil {
-			Path: ".",
+		return nil, err
-		}
-		if err := repo.Folder.Create(ctx, &f); err != nil {
-			return err
 	}
 
-		folderID = f.ID
+	if ret != nil {
-		return nil
+		return ret, nil
-	})
+	}
+
+	var parentID *file.FolderID
+
+	if p != "." {
+		parent := path.Dir(p)
+		parentFolder, err := getOrCreateFolder(ctx, parent)
+		if err != nil {
+			return nil, err
+		}
+
+		parentID = &parentFolder.ID
+	}
+
+	f := file.Folder{
+		Path:           p,
+		ParentFolderID: parentID,
+	}
+
+	if err := repo.Folder.Create(ctx, &f); err != nil {
+		return nil, err
+	}
+
+	ret = &f
+	return ret, nil
 }
 
 func makeTags(n int) {
@@ -230,11 +250,10 @@ func makePerformers(n int) {
 	}
 }
 
-func generateBaseFile(path string) *file.BaseFile {
+func generateBaseFile(parentFolderID file.FolderID, path string) *file.BaseFile {
 	return &file.BaseFile{
-		Path:           path,
 		Basename:       path,
-		ParentFolderID: folderID,
+		ParentFolderID: parentFolderID,
 		Fingerprints: []file.Fingerprint{
 			file.Fingerprint{
 				Type:        "md5",
@@ -250,11 +269,11 @@ func generateBaseFile(path string) *file.BaseFile {
 	}
 }
 
-func generateVideoFile(path string) file.File {
+func generateVideoFile(parentFolderID file.FolderID, path string) file.File {
 	w, h := getResolution()
 
 	return &file.VideoFile{
-		BaseFile: generateBaseFile(path),
+		BaseFile: generateBaseFile(parentFolderID, path),
 		Duration: rand.Float64() * 14400,
 		Height:   h,
 		Width:    w,
@@ -262,7 +281,13 @@ func generateVideoFile(path string) file.File {
 }
 
 func makeVideoFile(ctx context.Context, path string) (file.File, error) {
-	f := generateVideoFile(path)
+	folderPath := fsutil.GetIntraDir(path, 2, 2)
+	parentFolder, err := getOrCreateFolder(ctx, folderPath)
+	if err != nil {
+		return nil, err
+	}
+
+	f := generateVideoFile(parentFolder.ID, path)
 
 	if err := repo.File.Create(ctx, f); err != nil {
 		return nil, err
@@ -341,18 +366,24 @@ func generateScene(i int) models.Scene {
 	}
 }
 
-func generateImageFile(path string) file.File {
+func generateImageFile(parentFolderID file.FolderID, path string) file.File {
 	w, h := getResolution()
 
 	return &file.ImageFile{
-		BaseFile: generateBaseFile(path),
+		BaseFile: generateBaseFile(parentFolderID, path),
 		Height:   h,
 		Width:    w,
 	}
 }
 
 func makeImageFile(ctx context.Context, path string) (file.File, error) {
-	f := generateImageFile(path)
+	folderPath := fsutil.GetIntraDir(path, 2, 2)
+	parentFolder, err := getOrCreateFolder(ctx, folderPath)
+	if err != nil {
+		return nil, err
+	}
+
+	f := generateImageFile(parentFolder.ID, path)
 
 	if err := repo.File.Create(ctx, f); err != nil {
 		return nil, err
@@ -438,12 +469,18 @@ func makeGalleries(n int) {
 	}
 }
 
-func generateZipFile(path string) file.File {
+func generateZipFile(parentFolderID file.FolderID, path string) file.File {
-	return generateBaseFile(path)
+	return generateBaseFile(parentFolderID, path)
 }
 
 func makeZipFile(ctx context.Context, path string) (file.File, error) {
-	f := generateZipFile(path)
+	folderPath := fsutil.GetIntraDir(path, 2, 2)
+	parentFolder, err := getOrCreateFolder(ctx, folderPath)
+	if err != nil {
+		return nil, err
+	}
+
+	f := generateZipFile(parentFolder.ID, path)
 
 	if err := repo.File.Create(ctx, f); err != nil {
 		return nil, err

ui/v2.5/src/components/Scenes/SceneDetails/QueueViewer.tsx

@@ -1,7 +1,6 @@
 import React, { useEffect, useState } from "react";
 import { Link } from "react-router-dom";
 import cx from "classnames";
-import * as GQL from "src/core/generated-graphql";
 import { Button, Form, Spinner } from "react-bootstrap";
 import Icon from "src/components/Shared/Icon";
 import { useIntl } from "react-intl";
@@ -13,9 +12,10 @@ import {
   faStepForward,
 } from "@fortawesome/free-solid-svg-icons";
 import { objectTitle } from "src/core/files";
+import { QueuedScene } from "src/models/sceneQueue";
 
 export interface IPlaylistViewer {
-  scenes?: GQL.SlimSceneDataFragment[];
+  scenes?: QueuedScene[];
   currentID?: string;
   start?: number;
   continue?: boolean;
@@ -54,7 +54,7 @@ export const QueueViewer: React.FC<IPlaylistViewer> = ({
     setMoreLoading(false);
   }, [scenes]);
 
-  function isCurrentScene(scene: GQL.SlimSceneDataFragment) {
+  function isCurrentScene(scene: QueuedScene) {
     return scene.id === currentID;
   }
 
@@ -76,7 +76,7 @@ export const QueueViewer: React.FC<IPlaylistViewer> = ({
     onMoreScenes();
   }
 
-  function renderPlaylistEntry(scene: GQL.SlimSceneDataFragment) {
+  function renderPlaylistEntry(scene: QueuedScene) {
     return (
       <li
         className={cx("my-2", { current: isCurrentScene(scene) })}

ui/v2.5/src/components/Scenes/SceneDetails/Scene.tsx

@@ -19,7 +19,7 @@ import {
 
 import Icon from "src/components/Shared/Icon";
 import { useToast } from "src/hooks";
-import SceneQueue from "src/models/sceneQueue";
+import SceneQueue, { QueuedScene } from "src/models/sceneQueue";
 import { ListFilterModel } from "src/models/list-filter/filter";
 import Mousetrap from "mousetrap";
 import { OCounterButton } from "./OCounterButton";
@@ -56,7 +56,7 @@ interface IProps {
   scene: GQL.SceneDataFragment;
   refetch: () => void;
   setTimestamp: (num: number) => void;
-  queueScenes: GQL.SceneDataFragment[];
+  queueScenes: QueuedScene[];
   onQueueNext: () => void;
   onQueuePrevious: () => void;
   onQueueRandom: () => void;
@@ -519,7 +519,7 @@ const SceneLoader: React.FC = () => {
     () => SceneQueue.fromQueryParameters(location.search),
     [location.search]
   );
-  const [queueScenes, setQueueScenes] = useState<GQL.SceneDataFragment[]>([]);
+  const [queueScenes, setQueueScenes] = useState<QueuedScene[]>([]);
 
   const [queueTotal, setQueueTotal] = useState(0);
   const [queueStart, setQueueStart] = useState(1);
@@ -592,7 +592,7 @@ const SceneLoader: React.FC = () => {
     const { scenes } = query.data.findScenes;
 
     // prepend scenes to scene list
-    const newScenes = scenes.concat(queueScenes);
+    const newScenes = (scenes as QueuedScene[]).concat(queueScenes);
     setQueueScenes(newScenes);
     setQueueStart(newStart);
   }
@@ -613,7 +613,7 @@ const SceneLoader: React.FC = () => {
     const { scenes } = query.data.findScenes;
 
     // append scenes to scene list
-    const newScenes = scenes.concat(queueScenes);
+    const newScenes = (scenes as QueuedScene[]).concat(queueScenes);
     setQueueScenes(newScenes);
     // don't change queue start
   }

ui/v2.5/src/models/sceneQueue.ts

@@ -1,9 +1,11 @@
 import queryString from "query-string";
 import { RouteComponentProps } from "react-router-dom";
-import { FilterMode } from "src/core/generated-graphql";
+import { FilterMode, Scene } from "src/core/generated-graphql";
 import { ListFilterModel } from "./list-filter/filter";
 import { SceneListFilterOptions } from "./list-filter/scenes";
 
+export type QueuedScene = Pick<Scene, "id" | "title" | "paths">;
+
 interface IQueryParameters {
   qsort?: string;
   qsortd?: string;