mirror of
https://github.com/stashapp/stash.git
synced 2025-12-06 08:26:00 +01:00
[Files Refactor] Performance tuning (#2809)
* Use cache during migration * Avoid use of query views * Use FindMany to find related objects * Log slow queries * Add folders to generated files * Use SlimScene for scene queries * Include filename in migration error message
This commit is contained in:
WithoutPants
parent
c825cf5d09
commit
569c3a872a
21 changed files with 417 additions and 157 deletions
|
|
@ -4,7 +4,7 @@ query FindScenes($filter: FindFilterType, $scene_filter: SceneFilterType, $scene
|
|||
filesize
|
||||
duration
|
||||
scenes {
|
||||
...SceneData
|
||||
...SlimSceneData
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ func (r *galleryResolver) Date(ctx context.Context, obj *models.Gallery) (*strin
|
|||
func (r *galleryResolver) Scenes(ctx context.Context, obj *models.Gallery) (ret []*models.Scene, err error) {
|
||||
if err := r.withTxn(ctx, func(ctx context.Context) error {
|
||||
var err error
|
||||
ret, err = r.repository.Scene.FindByGalleryID(ctx, obj.ID)
|
||||
ret, err = r.repository.Scene.FindMany(ctx, obj.SceneIDs)
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -175,7 +175,7 @@ func (r *galleryResolver) Studio(ctx context.Context, obj *models.Gallery) (ret
|
|||
func (r *galleryResolver) Tags(ctx context.Context, obj *models.Gallery) (ret []*models.Tag, err error) {
|
||||
if err := r.withTxn(ctx, func(ctx context.Context) error {
|
||||
var err error
|
||||
ret, err = r.repository.Tag.FindByGalleryID(ctx, obj.ID)
|
||||
ret, err = r.repository.Tag.FindMany(ctx, obj.TagIDs)
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -187,7 +187,7 @@ func (r *galleryResolver) Tags(ctx context.Context, obj *models.Gallery) (ret []
|
|||
func (r *galleryResolver) Performers(ctx context.Context, obj *models.Gallery) (ret []*models.Performer, err error) {
|
||||
if err := r.withTxn(ctx, func(ctx context.Context) error {
|
||||
var err error
|
||||
ret, err = r.repository.Performer.FindByGalleryID(ctx, obj.ID)
|
||||
ret, err = r.repository.Performer.FindMany(ctx, obj.PerformerIDs)
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ func (r *sceneResolver) Captions(ctx context.Context, obj *models.Scene) (ret []
|
|||
|
||||
func (r *sceneResolver) Galleries(ctx context.Context, obj *models.Scene) (ret []*models.Gallery, err error) {
|
||||
if err := r.withTxn(ctx, func(ctx context.Context) error {
|
||||
ret, err = r.repository.Gallery.FindBySceneID(ctx, obj.ID)
|
||||
ret, err = r.repository.Gallery.FindMany(ctx, obj.GalleryIDs)
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -216,7 +216,7 @@ func (r *sceneResolver) Movies(ctx context.Context, obj *models.Scene) (ret []*S
|
|||
|
||||
func (r *sceneResolver) Tags(ctx context.Context, obj *models.Scene) (ret []*models.Tag, err error) {
|
||||
if err := r.withTxn(ctx, func(ctx context.Context) error {
|
||||
ret, err = r.repository.Tag.FindBySceneID(ctx, obj.ID)
|
||||
ret, err = r.repository.Tag.FindMany(ctx, obj.TagIDs)
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -227,7 +227,7 @@ func (r *sceneResolver) Tags(ctx context.Context, obj *models.Scene) (ret []*mod
|
|||
|
||||
func (r *sceneResolver) Performers(ctx context.Context, obj *models.Scene) (ret []*models.Performer, err error) {
|
||||
if err := r.withTxn(ctx, func(ctx context.Context) error {
|
||||
ret, err = r.repository.Performer.FindBySceneID(ctx, obj.ID)
|
||||
ret, err = r.repository.Performer.FindMany(ctx, obj.PerformerIDs)
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -517,7 +517,7 @@ func getPathSearchClause(pathColumn, basenameColumn, p string, addWildcards, not
|
|||
return ret
|
||||
}
|
||||
|
||||
func intCriterionHandler(c *models.IntCriterionInput, column string) criterionHandlerFunc {
|
||||
func intCriterionHandler(c *models.IntCriterionInput, column string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
|
||||
return func(ctx context.Context, f *filterBuilder) {
|
||||
if c != nil {
|
||||
clause, args := getIntCriterionWhereClause(column, *c)
|
||||
|
|
@ -526,9 +526,12 @@ func intCriterionHandler(c *models.IntCriterionInput, column string) criterionHa
|
|||
}
|
||||
}
|
||||
|
||||
func boolCriterionHandler(c *bool, column string) criterionHandlerFunc {
|
||||
func boolCriterionHandler(c *bool, column string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
|
||||
return func(ctx context.Context, f *filterBuilder) {
|
||||
if c != nil {
|
||||
if addJoinFn != nil {
|
||||
addJoinFn(f)
|
||||
}
|
||||
var v string
|
||||
if *c {
|
||||
v = "1"
|
||||
|
|
|
|||
|
|
@ -587,7 +587,8 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga
|
|||
|
||||
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
|
||||
if galleryFilter.Checksum != nil {
|
||||
f.addLeftJoin(fingerprintTable, "fingerprints_md5", "galleries_query.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
|
||||
qb.addGalleriesFilesTable(f)
|
||||
f.addLeftJoin(fingerprintTable, "fingerprints_md5", "galleries_files.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
|
||||
}
|
||||
|
||||
stringCriterionHandler(galleryFilter.Checksum, "fingerprints_md5.fingerprint")(ctx, f)
|
||||
|
|
@ -595,19 +596,21 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga
|
|||
|
||||
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
|
||||
if galleryFilter.IsZip != nil {
|
||||
qb.addGalleriesFilesTable(f)
|
||||
if *galleryFilter.IsZip {
|
||||
f.addWhere("galleries_query.file_id IS NOT NULL")
|
||||
|
||||
f.addWhere("galleries_files.file_id IS NOT NULL")
|
||||
} else {
|
||||
f.addWhere("galleries_query.file_id IS NULL")
|
||||
f.addWhere("galleries_files.file_id IS NULL")
|
||||
}
|
||||
}
|
||||
}))
|
||||
|
||||
query.handleCriterion(ctx, pathCriterionHandler(galleryFilter.Path, "galleries_query.parent_folder_path", "galleries_query.basename", nil))
|
||||
query.handleCriterion(ctx, pathCriterionHandler(galleryFilter.Path, "folders.path", "files.basename", qb.addFoldersTable))
|
||||
query.handleCriterion(ctx, galleryFileCountCriterionHandler(qb, galleryFilter.FileCount))
|
||||
query.handleCriterion(ctx, intCriterionHandler(galleryFilter.Rating, "galleries.rating"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(galleryFilter.Rating, "galleries.rating", nil))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(galleryFilter.URL, "galleries.url"))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(galleryFilter.Organized, "galleries.organized"))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(galleryFilter.Organized, "galleries.organized", nil))
|
||||
query.handleCriterion(ctx, galleryIsMissingCriterionHandler(qb, galleryFilter.IsMissing))
|
||||
query.handleCriterion(ctx, galleryTagsCriterionHandler(qb, galleryFilter.Tags))
|
||||
query.handleCriterion(ctx, galleryTagCountCriterionHandler(qb, galleryFilter.TagCount))
|
||||
|
|
@ -623,6 +626,20 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga
|
|||
return query
|
||||
}
|
||||
|
||||
func (qb *GalleryStore) addGalleriesFilesTable(f *filterBuilder) {
|
||||
f.addLeftJoin(galleriesFilesTable, "", "galleries_files.gallery_id = galleries.id")
|
||||
}
|
||||
|
||||
func (qb *GalleryStore) addFilesTable(f *filterBuilder) {
|
||||
qb.addGalleriesFilesTable(f)
|
||||
f.addLeftJoin(fileTable, "", "galleries_files.file_id = files.id")
|
||||
}
|
||||
|
||||
func (qb *GalleryStore) addFoldersTable(f *filterBuilder) {
|
||||
qb.addFilesTable(f)
|
||||
f.addLeftJoin(folderTable, "", "files.parent_folder_id = folders.id")
|
||||
}
|
||||
|
||||
func (qb *GalleryStore) makeQuery(ctx context.Context, galleryFilter *models.GalleryFilterType, findFilter *models.FindFilterType) (*queryBuilder, error) {
|
||||
if galleryFilter == nil {
|
||||
galleryFilter = &models.GalleryFilterType{}
|
||||
|
|
@ -634,16 +651,33 @@ func (qb *GalleryStore) makeQuery(ctx context.Context, galleryFilter *models.Gal
|
|||
query := qb.newQuery()
|
||||
distinctIDs(&query, galleryTable)
|
||||
|
||||
// for convenience, join with the query view
|
||||
query.addJoins(join{
|
||||
table: galleriesQueryTable.GetTable(),
|
||||
onClause: "galleries.id = galleries_query.id",
|
||||
joinType: "INNER",
|
||||
})
|
||||
|
||||
if q := findFilter.Q; q != nil && *q != "" {
|
||||
query.addJoins(
|
||||
join{
|
||||
table: galleriesFilesTable,
|
||||
onClause: "galleries_files.gallery_id = galleries.id",
|
||||
},
|
||||
join{
|
||||
table: fileTable,
|
||||
onClause: "galleries_files.file_id = files.id",
|
||||
},
|
||||
join{
|
||||
table: folderTable,
|
||||
onClause: "files.parent_folder_id = folders.id",
|
||||
},
|
||||
join{
|
||||
table: fingerprintTable,
|
||||
onClause: "files_fingerprints.file_id = galleries_files.file_id",
|
||||
},
|
||||
join{
|
||||
table: folderTable,
|
||||
as: "gallery_folder",
|
||||
onClause: "galleries.folder_id = gallery_folder.id",
|
||||
},
|
||||
)
|
||||
|
||||
// add joins for files and checksum
|
||||
searchColumns := []string{"galleries.title", "galleries_query.folder_path", "galleries_query.parent_folder_path", "galleries_query.basename", "galleries_query.fingerprint"}
|
||||
searchColumns := []string{"galleries.title", "gallery_folder.path", "folders.path", "files.basename", "files_fingerprints.fingerprint"}
|
||||
query.parseQueryString(searchColumns, *q)
|
||||
}
|
||||
|
||||
|
|
@ -654,7 +688,8 @@ func (qb *GalleryStore) makeQuery(ctx context.Context, galleryFilter *models.Gal
|
|||
|
||||
query.addFilter(filter)
|
||||
|
||||
query.sortAndPagination = qb.getGallerySort(findFilter) + getPagination(findFilter)
|
||||
qb.setGallerySort(&query, findFilter)
|
||||
query.sortAndPagination += getPagination(findFilter)
|
||||
|
||||
return &query, nil
|
||||
}
|
||||
|
|
@ -902,33 +937,52 @@ func galleryAverageResolutionCriterionHandler(qb *GalleryStore, resolution *mode
|
|||
}
|
||||
}
|
||||
|
||||
func (qb *GalleryStore) getGallerySort(findFilter *models.FindFilterType) string {
|
||||
func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.FindFilterType) {
|
||||
if findFilter == nil || findFilter.Sort == nil || *findFilter.Sort == "" {
|
||||
return ""
|
||||
return
|
||||
}
|
||||
|
||||
sort := findFilter.GetSort("path")
|
||||
direction := findFilter.GetDirection()
|
||||
|
||||
// translate sort field
|
||||
if sort == "file_mod_time" {
|
||||
sort = "mod_time"
|
||||
addFileTable := func() {
|
||||
query.addJoins(
|
||||
join{
|
||||
table: galleriesFilesTable,
|
||||
onClause: "galleries_files.gallery_id = galleries.id",
|
||||
},
|
||||
join{
|
||||
table: fileTable,
|
||||
onClause: "galleries_files.file_id = files.id",
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
switch sort {
|
||||
case "file_count":
|
||||
return getCountSort(galleryTable, galleriesFilesTable, galleryIDColumn, direction)
|
||||
query.sortAndPagination += getCountSort(galleryTable, galleriesFilesTable, galleryIDColumn, direction)
|
||||
case "images_count":
|
||||
return getCountSort(galleryTable, galleriesImagesTable, galleryIDColumn, direction)
|
||||
query.sortAndPagination += getCountSort(galleryTable, galleriesImagesTable, galleryIDColumn, direction)
|
||||
case "tag_count":
|
||||
return getCountSort(galleryTable, galleriesTagsTable, galleryIDColumn, direction)
|
||||
query.sortAndPagination += getCountSort(galleryTable, galleriesTagsTable, galleryIDColumn, direction)
|
||||
case "performer_count":
|
||||
return getCountSort(galleryTable, performersGalleriesTable, galleryIDColumn, direction)
|
||||
query.sortAndPagination += getCountSort(galleryTable, performersGalleriesTable, galleryIDColumn, direction)
|
||||
case "path":
|
||||
// special handling for path
|
||||
return fmt.Sprintf(" ORDER BY galleries_query.parent_folder_path %s, galleries_query.basename %[1]s", direction)
|
||||
addFileTable()
|
||||
query.addJoins(
|
||||
join{
|
||||
table: folderTable,
|
||||
onClause: "files.parent_folder_id = folders.id",
|
||||
},
|
||||
)
|
||||
query.sortAndPagination += fmt.Sprintf(" ORDER BY folders.path %s, files.basename %[1]s", direction)
|
||||
case "file_mod_time":
|
||||
sort = "mod_time"
|
||||
addFileTable()
|
||||
query.sortAndPagination += getSort(sort, direction, fileTable)
|
||||
default:
|
||||
return getSort(sort, direction, "galleries_query")
|
||||
query.sortAndPagination += getSort(sort, direction, "galleries")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -587,21 +587,21 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF
|
|||
|
||||
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
|
||||
if imageFilter.Checksum != nil {
|
||||
qb.addQueryTable(f)
|
||||
f.addInnerJoin(fingerprintTable, "fingerprints_md5", "galleries_query.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
|
||||
qb.addImagesFilesTable(f)
|
||||
f.addInnerJoin(fingerprintTable, "fingerprints_md5", "images_files.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
|
||||
}
|
||||
|
||||
stringCriterionHandler(imageFilter.Checksum, "fingerprints_md5.fingerprint")(ctx, f)
|
||||
}))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(imageFilter.Title, "images.title"))
|
||||
|
||||
query.handleCriterion(ctx, pathCriterionHandler(imageFilter.Path, "images_query.parent_folder_path", "images_query.basename", qb.addQueryTable))
|
||||
query.handleCriterion(ctx, pathCriterionHandler(imageFilter.Path, "folders.path", "files.basename", qb.addFoldersTable))
|
||||
query.handleCriterion(ctx, imageFileCountCriterionHandler(qb, imageFilter.FileCount))
|
||||
query.handleCriterion(ctx, intCriterionHandler(imageFilter.Rating, "images.rating"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(imageFilter.OCounter, "images.o_counter"))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(imageFilter.Organized, "images.organized"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(imageFilter.Rating, "images.rating", nil))
|
||||
query.handleCriterion(ctx, intCriterionHandler(imageFilter.OCounter, "images.o_counter", nil))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(imageFilter.Organized, "images.organized", nil))
|
||||
|
||||
query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "images_query.image_height", "images_query.image_width", qb.addQueryTable))
|
||||
query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "image_files.height", "image_files.width", qb.addImageFilesTable))
|
||||
query.handleCriterion(ctx, imageIsMissingCriterionHandler(qb, imageFilter.IsMissing))
|
||||
|
||||
query.handleCriterion(ctx, imageTagsCriterionHandler(qb, imageFilter.Tags))
|
||||
|
|
@ -616,8 +616,23 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF
|
|||
return query
|
||||
}
|
||||
|
||||
func (qb *ImageStore) addQueryTable(f *filterBuilder) {
|
||||
f.addInnerJoin(imagesQueryTable.GetTable(), "", "images.id = images_query.id")
|
||||
func (qb *ImageStore) addImagesFilesTable(f *filterBuilder) {
|
||||
f.addLeftJoin(imagesFilesTable, "", "images_files.image_id = images.id")
|
||||
}
|
||||
|
||||
func (qb *ImageStore) addFilesTable(f *filterBuilder) {
|
||||
qb.addImagesFilesTable(f)
|
||||
f.addLeftJoin(fileTable, "", "images_files.file_id = files.id")
|
||||
}
|
||||
|
||||
func (qb *ImageStore) addFoldersTable(f *filterBuilder) {
|
||||
qb.addFilesTable(f)
|
||||
f.addLeftJoin(folderTable, "", "files.parent_folder_id = folders.id")
|
||||
}
|
||||
|
||||
func (qb *ImageStore) addImageFilesTable(f *filterBuilder) {
|
||||
qb.addImagesFilesTable(f)
|
||||
f.addLeftJoin(imageFileTable, "", "image_files.file_id = images_files.file_id")
|
||||
}
|
||||
|
||||
func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) (*queryBuilder, error) {
|
||||
|
|
|
|||
|
|
@ -209,7 +209,7 @@ func (m *schema32Migrator) migrateFiles(ctx context.Context) error {
|
|||
|
||||
_, err = m.db.Exec("UPDATE `files` SET `parent_folder_id` = ?, `zip_file_id` = ?, `basename` = ? WHERE `id` = ?", parentID, zipFileID, basename, id)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("migrating file %s: %w", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -277,10 +277,23 @@ func (m *schema32Migrator) createFolderHierarchy(p string) (*int, sql.NullInt64,
|
|||
return m.getOrCreateFolder(p, nil, sql.NullInt64{})
|
||||
}
|
||||
|
||||
parentID, zipFileID, err := m.createFolderHierarchy(parent)
|
||||
var (
|
||||
parentID *int
|
||||
zipFileID sql.NullInt64
|
||||
err error
|
||||
)
|
||||
|
||||
// try to find parent folder in cache first
|
||||
foundEntry, ok := m.folderCache[parent]
|
||||
if ok {
|
||||
parentID = &foundEntry.id
|
||||
zipFileID = foundEntry.zipID
|
||||
} else {
|
||||
parentID, zipFileID, err = m.createFolderHierarchy(parent)
|
||||
if err != nil {
|
||||
return nil, sql.NullInt64{}, err
|
||||
}
|
||||
}
|
||||
|
||||
return m.getOrCreateFolder(p, parentID, zipFileID)
|
||||
}
|
||||
|
|
@ -323,12 +336,12 @@ func (m *schema32Migrator) getOrCreateFolder(path string, parentID *int, zipFile
|
|||
now := time.Now()
|
||||
result, err := m.db.Exec(insertSQL, path, parentFolderID, zipFileID, time.Time{}, now, now)
|
||||
if err != nil {
|
||||
return nil, sql.NullInt64{}, err
|
||||
return nil, sql.NullInt64{}, fmt.Errorf("creating folder %s: %w", path, err)
|
||||
}
|
||||
|
||||
id, err := result.LastInsertId()
|
||||
if err != nil {
|
||||
return nil, sql.NullInt64{}, err
|
||||
return nil, sql.NullInt64{}, fmt.Errorf("creating folder %s: %w", path, err)
|
||||
}
|
||||
|
||||
idInt := int(id)
|
||||
|
|
|
|||
|
|
@ -120,8 +120,8 @@ func (qb *movieQueryBuilder) makeFilter(ctx context.Context, movieFilter *models
|
|||
query.handleCriterion(ctx, stringCriterionHandler(movieFilter.Name, "movies.name"))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(movieFilter.Director, "movies.director"))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(movieFilter.Synopsis, "movies.synopsis"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(movieFilter.Rating, "movies.rating"))
|
||||
query.handleCriterion(ctx, durationCriterionHandler(movieFilter.Duration, "movies.duration"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(movieFilter.Rating, "movies.rating", nil))
|
||||
query.handleCriterion(ctx, durationCriterionHandler(movieFilter.Duration, "movies.duration", nil))
|
||||
query.handleCriterion(ctx, movieIsMissingCriterionHandler(qb, movieFilter.IsMissing))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(movieFilter.URL, "movies.url"))
|
||||
query.handleCriterion(ctx, movieStudioCriterionHandler(qb, movieFilter.Studios))
|
||||
|
|
|
|||
|
|
@ -245,8 +245,8 @@ func (qb *performerQueryBuilder) makeFilter(ctx context.Context, filter *models.
|
|||
query.handleCriterion(ctx, stringCriterionHandler(filter.Name, tableName+".name"))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(filter.Details, tableName+".details"))
|
||||
|
||||
query.handleCriterion(ctx, boolCriterionHandler(filter.FilterFavorites, tableName+".favorite"))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(filter.IgnoreAutoTag, tableName+".ignore_auto_tag"))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(filter.FilterFavorites, tableName+".favorite", nil))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(filter.IgnoreAutoTag, tableName+".ignore_auto_tag", nil))
|
||||
|
||||
query.handleCriterion(ctx, yearFilterCriterionHandler(filter.BirthYear, tableName+".birthdate"))
|
||||
query.handleCriterion(ctx, yearFilterCriterionHandler(filter.DeathYear, tableName+".death_date"))
|
||||
|
|
@ -269,10 +269,10 @@ func (qb *performerQueryBuilder) makeFilter(ctx context.Context, filter *models.
|
|||
query.handleCriterion(ctx, stringCriterionHandler(filter.CareerLength, tableName+".career_length"))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(filter.Tattoos, tableName+".tattoos"))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(filter.Piercings, tableName+".piercings"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(filter.Rating, tableName+".rating"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(filter.Rating, tableName+".rating", nil))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(filter.HairColor, tableName+".hair_color"))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(filter.URL, tableName+".url"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(filter.Weight, tableName+".weight"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(filter.Weight, tableName+".weight", nil))
|
||||
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
|
||||
if filter.StashID != nil {
|
||||
qb.stashIDRepository().join(f, "performer_stash_ids", "performers.id")
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
)
|
||||
|
||||
|
|
@ -58,7 +57,6 @@ func (qb queryBuilder) toSQL(includeSortPagination bool) string {
|
|||
func (qb queryBuilder) findIDs(ctx context.Context) ([]int, error) {
|
||||
const includeSortPagination = true
|
||||
sql := qb.toSQL(includeSortPagination)
|
||||
logger.Tracef("SQL: %s, args: %v", sql, qb.args)
|
||||
return qb.repository.runIdsQuery(ctx, sql, qb.args)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import (
|
|||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/stashapp/stash/pkg/file"
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
)
|
||||
|
||||
|
|
@ -154,8 +153,6 @@ func (r *repository) runIdsQuery(ctx context.Context, query string, args []inter
|
|||
}
|
||||
|
||||
func (r *repository) queryFunc(ctx context.Context, query string, args []interface{}, single bool, f func(rows *sqlx.Rows) error) error {
|
||||
logger.Tracef("SQL: %s, args: %v", query, args)
|
||||
|
||||
rows, err := r.tx.Queryx(ctx, query, args...)
|
||||
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
|
|
@ -252,8 +249,6 @@ func (r *repository) executeFindQuery(ctx context.Context, body string, args []i
|
|||
idsQuery := withClause + body + sortAndPagination
|
||||
|
||||
// Perform query and fetch result
|
||||
logger.Tracef("SQL: %s, args: %v", idsQuery, args)
|
||||
|
||||
var countResult int
|
||||
var countErr error
|
||||
var idsResult []int
|
||||
|
|
|
|||
|
|
@ -756,13 +756,14 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
|
|||
query.not(qb.makeFilter(ctx, sceneFilter.Not))
|
||||
}
|
||||
|
||||
query.handleCriterion(ctx, pathCriterionHandler(sceneFilter.Path, "scenes_query.parent_folder_path", "scenes_query.basename", nil))
|
||||
query.handleCriterion(ctx, pathCriterionHandler(sceneFilter.Path, "folders.path", "files.basename", qb.addFoldersTable))
|
||||
query.handleCriterion(ctx, sceneFileCountCriterionHandler(qb, sceneFilter.FileCount))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.Title, "scenes.title"))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.Details, "scenes.details"))
|
||||
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
|
||||
if sceneFilter.Oshash != nil {
|
||||
f.addLeftJoin(fingerprintTable, "fingerprints_oshash", "scenes_query.file_id = fingerprints_oshash.file_id AND fingerprints_oshash.type = 'oshash'")
|
||||
qb.addSceneFilesTable(f)
|
||||
f.addLeftJoin(fingerprintTable, "fingerprints_oshash", "scenes_files.file_id = fingerprints_oshash.file_id AND fingerprints_oshash.type = 'oshash'")
|
||||
}
|
||||
|
||||
stringCriterionHandler(sceneFilter.Oshash, "fingerprints_oshash.fingerprint")(ctx, f)
|
||||
|
|
@ -770,7 +771,8 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
|
|||
|
||||
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
|
||||
if sceneFilter.Checksum != nil {
|
||||
f.addLeftJoin(fingerprintTable, "fingerprints_md5", "scenes_query.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
|
||||
qb.addSceneFilesTable(f)
|
||||
f.addLeftJoin(fingerprintTable, "fingerprints_md5", "scenes_files.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
|
||||
}
|
||||
|
||||
stringCriterionHandler(sceneFilter.Checksum, "fingerprints_md5.fingerprint")(ctx, f)
|
||||
|
|
@ -778,22 +780,23 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
|
|||
|
||||
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
|
||||
if sceneFilter.Phash != nil {
|
||||
f.addLeftJoin(fingerprintTable, "fingerprints_phash", "scenes_query.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'")
|
||||
qb.addSceneFilesTable(f)
|
||||
f.addLeftJoin(fingerprintTable, "fingerprints_phash", "scenes_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'")
|
||||
|
||||
value, _ := utils.StringToPhash(sceneFilter.Phash.Value)
|
||||
intCriterionHandler(&models.IntCriterionInput{
|
||||
Value: int(value),
|
||||
Modifier: sceneFilter.Phash.Modifier,
|
||||
}, "fingerprints_phash.fingerprint")(ctx, f)
|
||||
}, "fingerprints_phash.fingerprint", nil)(ctx, f)
|
||||
}
|
||||
}))
|
||||
|
||||
query.handleCriterion(ctx, intCriterionHandler(sceneFilter.Rating, "scenes.rating"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(sceneFilter.OCounter, "scenes.o_counter"))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(sceneFilter.Organized, "scenes.organized"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(sceneFilter.Rating, "scenes.rating", nil))
|
||||
query.handleCriterion(ctx, intCriterionHandler(sceneFilter.OCounter, "scenes.o_counter", nil))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(sceneFilter.Organized, "scenes.organized", nil))
|
||||
|
||||
query.handleCriterion(ctx, durationCriterionHandler(sceneFilter.Duration, "scenes_query.duration"))
|
||||
query.handleCriterion(ctx, resolutionCriterionHandler(sceneFilter.Resolution, "scenes_query.video_height", "scenes_query.video_width", nil))
|
||||
query.handleCriterion(ctx, durationCriterionHandler(sceneFilter.Duration, "video_files.duration", qb.addVideoFilesTable))
|
||||
query.handleCriterion(ctx, resolutionCriterionHandler(sceneFilter.Resolution, "video_files.height", "video_files.width", qb.addVideoFilesTable))
|
||||
|
||||
query.handleCriterion(ctx, hasMarkersCriterionHandler(sceneFilter.HasMarkers))
|
||||
query.handleCriterion(ctx, sceneIsMissingCriterionHandler(qb, sceneFilter.IsMissing))
|
||||
|
|
@ -806,8 +809,8 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
|
|||
}
|
||||
}))
|
||||
|
||||
query.handleCriterion(ctx, boolCriterionHandler(sceneFilter.Interactive, "scenes.interactive"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(sceneFilter.InteractiveSpeed, "scenes.interactive_speed"))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(sceneFilter.Interactive, "video_files.interactive", qb.addVideoFilesTable))
|
||||
query.handleCriterion(ctx, intCriterionHandler(sceneFilter.InteractiveSpeed, "video_files.interactive_speed", qb.addVideoFilesTable))
|
||||
|
||||
query.handleCriterion(ctx, sceneCaptionCriterionHandler(qb, sceneFilter.Captions))
|
||||
|
||||
|
|
@ -820,11 +823,30 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
|
|||
query.handleCriterion(ctx, scenePerformerTagsCriterionHandler(qb, sceneFilter.PerformerTags))
|
||||
query.handleCriterion(ctx, scenePerformerFavoriteCriterionHandler(sceneFilter.PerformerFavorite))
|
||||
query.handleCriterion(ctx, scenePerformerAgeCriterionHandler(sceneFilter.PerformerAge))
|
||||
query.handleCriterion(ctx, scenePhashDuplicatedCriterionHandler(sceneFilter.Duplicated))
|
||||
query.handleCriterion(ctx, scenePhashDuplicatedCriterionHandler(sceneFilter.Duplicated, qb.addSceneFilesTable))
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
func (qb *SceneStore) addSceneFilesTable(f *filterBuilder) {
|
||||
f.addLeftJoin(scenesFilesTable, "", "scenes_files.scene_id = scenes.id")
|
||||
}
|
||||
|
||||
func (qb *SceneStore) addFilesTable(f *filterBuilder) {
|
||||
qb.addSceneFilesTable(f)
|
||||
f.addLeftJoin(fileTable, "", "scenes_files.file_id = files.id")
|
||||
}
|
||||
|
||||
func (qb *SceneStore) addFoldersTable(f *filterBuilder) {
|
||||
qb.addFilesTable(f)
|
||||
f.addLeftJoin(folderTable, "", "files.parent_folder_id = folders.id")
|
||||
}
|
||||
|
||||
func (qb *SceneStore) addVideoFilesTable(f *filterBuilder) {
|
||||
qb.addSceneFilesTable(f)
|
||||
f.addLeftJoin(videoFileTable, "", "video_files.file_id = scenes_files.file_id")
|
||||
}
|
||||
|
||||
func (qb *SceneStore) Query(ctx context.Context, options models.SceneQueryOptions) (*models.SceneQueryResult, error) {
|
||||
sceneFilter := options.SceneFilter
|
||||
findFilter := options.FindFilter
|
||||
|
|
@ -839,17 +861,31 @@ func (qb *SceneStore) Query(ctx context.Context, options models.SceneQueryOption
|
|||
query := qb.newQuery()
|
||||
distinctIDs(&query, sceneTable)
|
||||
|
||||
// for convenience, join with the query view
|
||||
query.addJoins(join{
|
||||
table: scenesQueryTable.GetTable(),
|
||||
onClause: "scenes.id = scenes_query.id",
|
||||
joinType: "INNER",
|
||||
})
|
||||
|
||||
if q := findFilter.Q; q != nil && *q != "" {
|
||||
query.join("scene_markers", "", "scene_markers.scene_id = scenes.id")
|
||||
query.addJoins(
|
||||
join{
|
||||
table: scenesFilesTable,
|
||||
onClause: "scenes_files.scene_id = scenes.id",
|
||||
},
|
||||
join{
|
||||
table: fileTable,
|
||||
onClause: "scenes_files.file_id = files.id",
|
||||
},
|
||||
join{
|
||||
table: folderTable,
|
||||
onClause: "files.parent_folder_id = folders.id",
|
||||
},
|
||||
join{
|
||||
table: fingerprintTable,
|
||||
onClause: "files_fingerprints.file_id = scenes_files.file_id",
|
||||
},
|
||||
join{
|
||||
table: sceneMarkerTable,
|
||||
onClause: "scene_markers.scene_id = scenes.id",
|
||||
},
|
||||
)
|
||||
|
||||
searchColumns := []string{"scenes.title", "scenes.details", "scenes_query.parent_folder_path", "scenes_query.basename", "scenes_query.fingerprint", "scene_markers.title"}
|
||||
searchColumns := []string{"scenes.title", "scenes.details", "folders.path", "files.basename", "files_fingerprints.fingerprint", "scene_markers.title"}
|
||||
query.parseQueryString(searchColumns, *q)
|
||||
}
|
||||
|
||||
|
|
@ -890,12 +926,32 @@ func (qb *SceneStore) queryGroupedFields(ctx context.Context, options models.Sce
|
|||
}
|
||||
|
||||
if options.TotalDuration {
|
||||
query.addColumn("COALESCE(scenes_query.duration, 0) as duration")
|
||||
query.addJoins(
|
||||
join{
|
||||
table: scenesFilesTable,
|
||||
onClause: "scenes_files.scene_id = scenes.id",
|
||||
},
|
||||
join{
|
||||
table: videoFileTable,
|
||||
onClause: "scenes_files.file_id = video_files.file_id",
|
||||
},
|
||||
)
|
||||
query.addColumn("COALESCE(video_files.duration, 0) as duration")
|
||||
aggregateQuery.addColumn("SUM(temp.duration) as duration")
|
||||
}
|
||||
|
||||
if options.TotalSize {
|
||||
query.addColumn("COALESCE(scenes_query.size, 0) as size")
|
||||
query.addJoins(
|
||||
join{
|
||||
table: scenesFilesTable,
|
||||
onClause: "scenes_files.scene_id = scenes.id",
|
||||
},
|
||||
join{
|
||||
table: fileTable,
|
||||
onClause: "scenes_files.file_id = files.id",
|
||||
},
|
||||
)
|
||||
query.addColumn("COALESCE(files.size, 0) as size")
|
||||
aggregateQuery.addColumn("SUM(temp.size) as size")
|
||||
}
|
||||
|
||||
|
|
@ -928,24 +984,32 @@ func sceneFileCountCriterionHandler(qb *SceneStore, fileCount *models.IntCriteri
|
|||
return h.handler(fileCount)
|
||||
}
|
||||
|
||||
func scenePhashDuplicatedCriterionHandler(duplicatedFilter *models.PHashDuplicationCriterionInput) criterionHandlerFunc {
|
||||
func scenePhashDuplicatedCriterionHandler(duplicatedFilter *models.PHashDuplicationCriterionInput, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
|
||||
return func(ctx context.Context, f *filterBuilder) {
|
||||
// TODO: Wishlist item: Implement Distance matching
|
||||
if duplicatedFilter != nil {
|
||||
if addJoinFn != nil {
|
||||
addJoinFn(f)
|
||||
}
|
||||
|
||||
var v string
|
||||
if *duplicatedFilter.Duplicated {
|
||||
v = ">"
|
||||
} else {
|
||||
v = "="
|
||||
}
|
||||
f.addInnerJoin("(SELECT file_id FROM files_fingerprints INNER JOIN (SELECT fingerprint FROM files_fingerprints WHERE type = 'phash' GROUP BY fingerprint HAVING COUNT (fingerprint) "+v+" 1) dupes on files_fingerprints.fingerprint = dupes.fingerprint)", "scph", "scenes_query.file_id = scph.file_id")
|
||||
|
||||
f.addInnerJoin("(SELECT file_id FROM files_fingerprints INNER JOIN (SELECT fingerprint FROM files_fingerprints WHERE type = 'phash' GROUP BY fingerprint HAVING COUNT (fingerprint) "+v+" 1) dupes on files_fingerprints.fingerprint = dupes.fingerprint)", "scph", "scenes_files.file_id = scph.file_id")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func durationCriterionHandler(durationFilter *models.IntCriterionInput, column string) criterionHandlerFunc {
|
||||
func durationCriterionHandler(durationFilter *models.IntCriterionInput, column string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc {
|
||||
return func(ctx context.Context, f *filterBuilder) {
|
||||
if durationFilter != nil {
|
||||
if addJoinFn != nil {
|
||||
addJoinFn(f)
|
||||
}
|
||||
clause, args := getIntCriterionWhereClause("cast("+column+" as int)", *durationFilter)
|
||||
f.addWhere(clause, args...)
|
||||
}
|
||||
|
|
@ -1015,7 +1079,8 @@ func sceneIsMissingCriterionHandler(qb *SceneStore, isMissing *string) criterion
|
|||
qb.stashIDRepository().join(f, "scene_stash_ids", "scenes.id")
|
||||
f.addWhere("scene_stash_ids.scene_id IS NULL")
|
||||
case "phash":
|
||||
f.addLeftJoin(fingerprintTable, "fingerprints_phash", "scenes_query.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'")
|
||||
qb.addSceneFilesTable(f)
|
||||
f.addLeftJoin(fingerprintTable, "fingerprints_phash", "scenes_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'")
|
||||
f.addWhere("fingerprints_phash.fingerprint IS NULL")
|
||||
default:
|
||||
f.addWhere("(scenes." + *isMissing + " IS NULL OR TRIM(scenes." + *isMissing + ") = '')")
|
||||
|
|
@ -1040,7 +1105,8 @@ func sceneCaptionCriterionHandler(qb *SceneStore, captions *models.StringCriteri
|
|||
joinTable: videoCaptionsTable,
|
||||
stringColumn: captionCodeColumn,
|
||||
addJoinTable: func(f *filterBuilder) {
|
||||
f.addLeftJoin(videoCaptionsTable, "", "video_captions.file_id = scenes_query.file_id")
|
||||
qb.addSceneFilesTable(f)
|
||||
f.addLeftJoin(videoCaptionsTable, "", "video_captions.file_id = scenes_files.file_id")
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -1201,14 +1267,27 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF
|
|||
}
|
||||
sort := findFilter.GetSort("title")
|
||||
|
||||
// translate sort field
|
||||
switch sort {
|
||||
case "bitrate":
|
||||
sort = "bit_rate"
|
||||
case "file_mod_time":
|
||||
sort = "mod_time"
|
||||
case "framerate":
|
||||
sort = "frame_rate"
|
||||
addFileTable := func() {
|
||||
query.addJoins(
|
||||
join{
|
||||
table: scenesFilesTable,
|
||||
onClause: "scenes_files.scene_id = scenes.id",
|
||||
},
|
||||
join{
|
||||
table: fileTable,
|
||||
onClause: "scenes_files.file_id = files.id",
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
addVideoFileTable := func() {
|
||||
addFileTable()
|
||||
query.addJoins(
|
||||
join{
|
||||
table: videoFileTable,
|
||||
onClause: "video_files.file_id = scenes_files.file_id",
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
direction := findFilter.GetDirection()
|
||||
|
|
@ -1224,21 +1303,47 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF
|
|||
query.sortAndPagination += getCountSort(sceneTable, scenesFilesTable, sceneIDColumn, direction)
|
||||
case "path":
|
||||
// special handling for path
|
||||
query.sortAndPagination += fmt.Sprintf(" ORDER BY scenes_query.parent_folder_path %s, scenes_query.basename %[1]s", direction)
|
||||
addFileTable()
|
||||
query.addJoins(
|
||||
join{
|
||||
table: folderTable,
|
||||
onClause: "files.parent_folder_id = folders.id",
|
||||
},
|
||||
)
|
||||
query.sortAndPagination += fmt.Sprintf(" ORDER BY folders.path %s, files.basename %[1]s", direction)
|
||||
case "perceptual_similarity":
|
||||
// special handling for phash
|
||||
query.addJoins(join{
|
||||
addFileTable()
|
||||
query.addJoins(
|
||||
join{
|
||||
table: fingerprintTable,
|
||||
as: "fingerprints_phash",
|
||||
onClause: "scenes_query.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'",
|
||||
})
|
||||
onClause: "scenes_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'",
|
||||
},
|
||||
)
|
||||
|
||||
query.sortAndPagination += " ORDER BY fingerprints_phash.fingerprint " + direction + ", scenes_query.size DESC"
|
||||
query.sortAndPagination += " ORDER BY fingerprints_phash.fingerprint " + direction + ", files.size DESC"
|
||||
case "bitrate":
|
||||
sort = "bit_rate"
|
||||
addVideoFileTable()
|
||||
query.sortAndPagination += getSort(sort, direction, videoFileTable)
|
||||
case "file_mod_time":
|
||||
sort = "mod_time"
|
||||
addFileTable()
|
||||
query.sortAndPagination += getSort(sort, direction, fileTable)
|
||||
case "framerate":
|
||||
sort = "frame_rate"
|
||||
addVideoFileTable()
|
||||
query.sortAndPagination += getSort(sort, direction, videoFileTable)
|
||||
case "size":
|
||||
addFileTable()
|
||||
query.sortAndPagination += getSort(sort, direction, fileTable)
|
||||
case "duration":
|
||||
addVideoFileTable()
|
||||
query.sortAndPagination += getSort(sort, direction, videoFileTable)
|
||||
default:
|
||||
query.sortAndPagination += getSort(sort, direction, "scenes_query")
|
||||
query.sortAndPagination += getSort(sort, direction, "scenes")
|
||||
}
|
||||
|
||||
query.sortAndPagination += ", scenes_query.bit_rate DESC, scenes_query.frame_rate DESC, scenes.rating DESC, scenes_query.duration DESC"
|
||||
}
|
||||
|
||||
func (qb *SceneStore) imageRepository() *imageRepository {
|
||||
|
|
|
|||
|
|
@ -1853,8 +1853,11 @@ func queryScene(ctx context.Context, t *testing.T, sqb models.SceneReader, scene
|
|||
result, err := sqb.Query(ctx, models.SceneQueryOptions{
|
||||
QueryOptions: models.QueryOptions{
|
||||
FindFilter: findFilter,
|
||||
Count: true,
|
||||
},
|
||||
SceneFilter: sceneFilter,
|
||||
TotalDuration: true,
|
||||
TotalSize: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("Error querying scene: %v", err)
|
||||
|
|
@ -1875,7 +1878,9 @@ func sceneQueryQ(ctx context.Context, t *testing.T, sqb models.SceneReader, q st
|
|||
}
|
||||
scenes := queryScene(ctx, t, sqb, nil, &filter)
|
||||
|
||||
assert.Len(t, scenes, 1)
|
||||
if !assert.Len(t, scenes, 1) {
|
||||
return
|
||||
}
|
||||
scene := scenes[0]
|
||||
assert.Equal(t, sceneIDs[expectedSceneIdx], scene.ID)
|
||||
|
||||
|
|
|
|||
|
|
@ -207,8 +207,8 @@ func (qb *studioQueryBuilder) makeFilter(ctx context.Context, studioFilter *mode
|
|||
query.handleCriterion(ctx, stringCriterionHandler(studioFilter.Name, studioTable+".name"))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(studioFilter.Details, studioTable+".details"))
|
||||
query.handleCriterion(ctx, stringCriterionHandler(studioFilter.URL, studioTable+".url"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(studioFilter.Rating, studioTable+".rating"))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(studioFilter.IgnoreAutoTag, studioTable+".ignore_auto_tag"))
|
||||
query.handleCriterion(ctx, intCriterionHandler(studioFilter.Rating, studioTable+".rating", nil))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(studioFilter.IgnoreAutoTag, studioTable+".ignore_auto_tag", nil))
|
||||
|
||||
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
|
||||
if studioFilter.StashID != nil {
|
||||
|
|
|
|||
|
|
@ -557,7 +557,6 @@ func queryFunc(ctx context.Context, query *goqu.SelectDataset, single bool, f fu
|
|||
return err
|
||||
}
|
||||
|
||||
logger.Tracef("SQL: %s [%v]", q, args)
|
||||
rows, err := tx.QueryxContext(ctx, q, args...)
|
||||
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
|
|
@ -592,7 +591,6 @@ func querySimple(ctx context.Context, query *goqu.SelectDataset, out interface{}
|
|||
return err
|
||||
}
|
||||
|
||||
logger.Tracef("SQL: %s [%v]", q, args)
|
||||
rows, err := tx.QueryxContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("querying `%s` [%v]: %w", q, args, err)
|
||||
|
|
|
|||
|
|
@ -297,7 +297,7 @@ func (qb *tagQueryBuilder) makeFilter(ctx context.Context, tagFilter *models.Tag
|
|||
|
||||
query.handleCriterion(ctx, stringCriterionHandler(tagFilter.Name, tagTable+".name"))
|
||||
query.handleCriterion(ctx, tagAliasCriterionHandler(qb, tagFilter.Aliases))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(tagFilter.IgnoreAutoTag, tagTable+".ignore_auto_tag"))
|
||||
query.handleCriterion(ctx, boolCriterionHandler(tagFilter.IgnoreAutoTag, tagTable+".ignore_auto_tag", nil))
|
||||
|
||||
query.handleCriterion(ctx, tagIsMissingCriterionHandler(qb, tagFilter.IsMissing))
|
||||
query.handleCriterion(ctx, tagSceneCountCriterionHandler(qb, tagFilter.SceneCount))
|
||||
|
|
|
|||
|
|
@ -3,8 +3,14 @@ package sqlite
|
|||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
slowLogTime = time.Millisecond * 200
|
||||
)
|
||||
|
||||
type dbReader interface {
|
||||
|
|
@ -14,6 +20,15 @@ type dbReader interface {
|
|||
QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error)
|
||||
}
|
||||
|
||||
func logSQL(start time.Time, query string, args ...interface{}) {
|
||||
since := time.Since(start)
|
||||
if since >= slowLogTime {
|
||||
logger.Debugf("SLOW SQL [%v]: %s, args: %v", since, query, args)
|
||||
} else {
|
||||
logger.Tracef("SQL [%v]: %s, args: %v", since, query, args)
|
||||
}
|
||||
}
|
||||
|
||||
type dbWrapper struct{}
|
||||
|
||||
func (*dbWrapper) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
|
||||
|
|
@ -22,7 +37,11 @@ func (*dbWrapper) Get(ctx context.Context, dest interface{}, query string, args
|
|||
return err
|
||||
}
|
||||
|
||||
return tx.Get(dest, query, args...)
|
||||
start := time.Now()
|
||||
err = tx.Get(dest, query, args...)
|
||||
logSQL(start, query, args...)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (*dbWrapper) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
|
||||
|
|
@ -31,7 +50,11 @@ func (*dbWrapper) Select(ctx context.Context, dest interface{}, query string, ar
|
|||
return err
|
||||
}
|
||||
|
||||
return tx.Select(dest, query, args...)
|
||||
start := time.Now()
|
||||
err = tx.Select(dest, query, args...)
|
||||
logSQL(start, query, args...)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (*dbWrapper) Queryx(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) {
|
||||
|
|
@ -40,7 +63,11 @@ func (*dbWrapper) Queryx(ctx context.Context, query string, args ...interface{})
|
|||
return nil, err
|
||||
}
|
||||
|
||||
return tx.Queryx(query, args...)
|
||||
start := time.Now()
|
||||
ret, err := tx.Queryx(query, args...)
|
||||
logSQL(start, query, args...)
|
||||
|
||||
return ret, err
|
||||
}
|
||||
|
||||
func (*dbWrapper) NamedExec(ctx context.Context, query string, arg interface{}) (sql.Result, error) {
|
||||
|
|
@ -49,7 +76,11 @@ func (*dbWrapper) NamedExec(ctx context.Context, query string, arg interface{})
|
|||
return nil, err
|
||||
}
|
||||
|
||||
return tx.NamedExec(query, arg)
|
||||
start := time.Now()
|
||||
ret, err := tx.NamedExec(query, arg)
|
||||
logSQL(start, query, arg)
|
||||
|
||||
return ret, err
|
||||
}
|
||||
|
||||
func (*dbWrapper) Exec(ctx context.Context, query string, args ...interface{}) (sql.Result, error) {
|
||||
|
|
@ -58,5 +89,9 @@ func (*dbWrapper) Exec(ctx context.Context, query string, args ...interface{}) (
|
|||
return nil, err
|
||||
}
|
||||
|
||||
return tx.Exec(query, args...)
|
||||
start := time.Now()
|
||||
ret, err := tx.Exec(query, args...)
|
||||
logSQL(start, query, args...)
|
||||
|
||||
return ret, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,10 +11,12 @@ import (
|
|||
"math"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/stashapp/stash/pkg/file"
|
||||
"github.com/stashapp/stash/pkg/fsutil"
|
||||
"github.com/stashapp/stash/pkg/hash/md5"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
"github.com/stashapp/stash/pkg/sliceutil/intslice"
|
||||
|
|
@ -66,9 +68,6 @@ func main() {
|
|||
log.Fatalf("couldn't initialize database: %v", err)
|
||||
}
|
||||
logf("Populating database...")
|
||||
if err = makeFolder(); err != nil {
|
||||
log.Fatalf("couldn't create folder: %v", err)
|
||||
}
|
||||
populateDB()
|
||||
}
|
||||
|
||||
|
|
@ -117,18 +116,39 @@ func retry(attempts int, fn func() error) error {
|
|||
return err
|
||||
}
|
||||
|
||||
func makeFolder() error {
|
||||
return withTxn(func(ctx context.Context) error {
|
||||
f := file.Folder{
|
||||
Path: ".",
|
||||
}
|
||||
if err := repo.Folder.Create(ctx, &f); err != nil {
|
||||
return err
|
||||
func getOrCreateFolder(ctx context.Context, p string) (*file.Folder, error) {
|
||||
ret, err := repo.Folder.FindByPath(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
folderID = f.ID
|
||||
return nil
|
||||
})
|
||||
if ret != nil {
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
var parentID *file.FolderID
|
||||
|
||||
if p != "." {
|
||||
parent := path.Dir(p)
|
||||
parentFolder, err := getOrCreateFolder(ctx, parent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parentID = &parentFolder.ID
|
||||
}
|
||||
|
||||
f := file.Folder{
|
||||
Path: p,
|
||||
ParentFolderID: parentID,
|
||||
}
|
||||
|
||||
if err := repo.Folder.Create(ctx, &f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ret = &f
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func makeTags(n int) {
|
||||
|
|
@ -230,11 +250,10 @@ func makePerformers(n int) {
|
|||
}
|
||||
}
|
||||
|
||||
func generateBaseFile(path string) *file.BaseFile {
|
||||
func generateBaseFile(parentFolderID file.FolderID, path string) *file.BaseFile {
|
||||
return &file.BaseFile{
|
||||
Path: path,
|
||||
Basename: path,
|
||||
ParentFolderID: folderID,
|
||||
ParentFolderID: parentFolderID,
|
||||
Fingerprints: []file.Fingerprint{
|
||||
file.Fingerprint{
|
||||
Type: "md5",
|
||||
|
|
@ -250,11 +269,11 @@ func generateBaseFile(path string) *file.BaseFile {
|
|||
}
|
||||
}
|
||||
|
||||
func generateVideoFile(path string) file.File {
|
||||
func generateVideoFile(parentFolderID file.FolderID, path string) file.File {
|
||||
w, h := getResolution()
|
||||
|
||||
return &file.VideoFile{
|
||||
BaseFile: generateBaseFile(path),
|
||||
BaseFile: generateBaseFile(parentFolderID, path),
|
||||
Duration: rand.Float64() * 14400,
|
||||
Height: h,
|
||||
Width: w,
|
||||
|
|
@ -262,7 +281,13 @@ func generateVideoFile(path string) file.File {
|
|||
}
|
||||
|
||||
func makeVideoFile(ctx context.Context, path string) (file.File, error) {
|
||||
f := generateVideoFile(path)
|
||||
folderPath := fsutil.GetIntraDir(path, 2, 2)
|
||||
parentFolder, err := getOrCreateFolder(ctx, folderPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f := generateVideoFile(parentFolder.ID, path)
|
||||
|
||||
if err := repo.File.Create(ctx, f); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -341,18 +366,24 @@ func generateScene(i int) models.Scene {
|
|||
}
|
||||
}
|
||||
|
||||
func generateImageFile(path string) file.File {
|
||||
func generateImageFile(parentFolderID file.FolderID, path string) file.File {
|
||||
w, h := getResolution()
|
||||
|
||||
return &file.ImageFile{
|
||||
BaseFile: generateBaseFile(path),
|
||||
BaseFile: generateBaseFile(parentFolderID, path),
|
||||
Height: h,
|
||||
Width: w,
|
||||
}
|
||||
}
|
||||
|
||||
func makeImageFile(ctx context.Context, path string) (file.File, error) {
|
||||
f := generateImageFile(path)
|
||||
folderPath := fsutil.GetIntraDir(path, 2, 2)
|
||||
parentFolder, err := getOrCreateFolder(ctx, folderPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f := generateImageFile(parentFolder.ID, path)
|
||||
|
||||
if err := repo.File.Create(ctx, f); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -438,12 +469,18 @@ func makeGalleries(n int) {
|
|||
}
|
||||
}
|
||||
|
||||
func generateZipFile(path string) file.File {
|
||||
return generateBaseFile(path)
|
||||
func generateZipFile(parentFolderID file.FolderID, path string) file.File {
|
||||
return generateBaseFile(parentFolderID, path)
|
||||
}
|
||||
|
||||
func makeZipFile(ctx context.Context, path string) (file.File, error) {
|
||||
f := generateZipFile(path)
|
||||
folderPath := fsutil.GetIntraDir(path, 2, 2)
|
||||
parentFolder, err := getOrCreateFolder(ctx, folderPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
f := generateZipFile(parentFolder.ID, path)
|
||||
|
||||
if err := repo.File.Create(ctx, f); err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import React, { useEffect, useState } from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
import cx from "classnames";
|
||||
import * as GQL from "src/core/generated-graphql";
|
||||
import { Button, Form, Spinner } from "react-bootstrap";
|
||||
import Icon from "src/components/Shared/Icon";
|
||||
import { useIntl } from "react-intl";
|
||||
|
|
@ -13,9 +12,10 @@ import {
|
|||
faStepForward,
|
||||
} from "@fortawesome/free-solid-svg-icons";
|
||||
import { objectTitle } from "src/core/files";
|
||||
import { QueuedScene } from "src/models/sceneQueue";
|
||||
|
||||
export interface IPlaylistViewer {
|
||||
scenes?: GQL.SlimSceneDataFragment[];
|
||||
scenes?: QueuedScene[];
|
||||
currentID?: string;
|
||||
start?: number;
|
||||
continue?: boolean;
|
||||
|
|
@ -54,7 +54,7 @@ export const QueueViewer: React.FC<IPlaylistViewer> = ({
|
|||
setMoreLoading(false);
|
||||
}, [scenes]);
|
||||
|
||||
function isCurrentScene(scene: GQL.SlimSceneDataFragment) {
|
||||
function isCurrentScene(scene: QueuedScene) {
|
||||
return scene.id === currentID;
|
||||
}
|
||||
|
||||
|
|
@ -76,7 +76,7 @@ export const QueueViewer: React.FC<IPlaylistViewer> = ({
|
|||
onMoreScenes();
|
||||
}
|
||||
|
||||
function renderPlaylistEntry(scene: GQL.SlimSceneDataFragment) {
|
||||
function renderPlaylistEntry(scene: QueuedScene) {
|
||||
return (
|
||||
<li
|
||||
className={cx("my-2", { current: isCurrentScene(scene) })}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import {
|
|||
|
||||
import Icon from "src/components/Shared/Icon";
|
||||
import { useToast } from "src/hooks";
|
||||
import SceneQueue from "src/models/sceneQueue";
|
||||
import SceneQueue, { QueuedScene } from "src/models/sceneQueue";
|
||||
import { ListFilterModel } from "src/models/list-filter/filter";
|
||||
import Mousetrap from "mousetrap";
|
||||
import { OCounterButton } from "./OCounterButton";
|
||||
|
|
@ -56,7 +56,7 @@ interface IProps {
|
|||
scene: GQL.SceneDataFragment;
|
||||
refetch: () => void;
|
||||
setTimestamp: (num: number) => void;
|
||||
queueScenes: GQL.SceneDataFragment[];
|
||||
queueScenes: QueuedScene[];
|
||||
onQueueNext: () => void;
|
||||
onQueuePrevious: () => void;
|
||||
onQueueRandom: () => void;
|
||||
|
|
@ -519,7 +519,7 @@ const SceneLoader: React.FC = () => {
|
|||
() => SceneQueue.fromQueryParameters(location.search),
|
||||
[location.search]
|
||||
);
|
||||
const [queueScenes, setQueueScenes] = useState<GQL.SceneDataFragment[]>([]);
|
||||
const [queueScenes, setQueueScenes] = useState<QueuedScene[]>([]);
|
||||
|
||||
const [queueTotal, setQueueTotal] = useState(0);
|
||||
const [queueStart, setQueueStart] = useState(1);
|
||||
|
|
@ -592,7 +592,7 @@ const SceneLoader: React.FC = () => {
|
|||
const { scenes } = query.data.findScenes;
|
||||
|
||||
// prepend scenes to scene list
|
||||
const newScenes = scenes.concat(queueScenes);
|
||||
const newScenes = (scenes as QueuedScene[]).concat(queueScenes);
|
||||
setQueueScenes(newScenes);
|
||||
setQueueStart(newStart);
|
||||
}
|
||||
|
|
@ -613,7 +613,7 @@ const SceneLoader: React.FC = () => {
|
|||
const { scenes } = query.data.findScenes;
|
||||
|
||||
// append scenes to scene list
|
||||
const newScenes = scenes.concat(queueScenes);
|
||||
const newScenes = (scenes as QueuedScene[]).concat(queueScenes);
|
||||
setQueueScenes(newScenes);
|
||||
// don't change queue start
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import queryString from "query-string";
|
||||
import { RouteComponentProps } from "react-router-dom";
|
||||
import { FilterMode } from "src/core/generated-graphql";
|
||||
import { FilterMode, Scene } from "src/core/generated-graphql";
|
||||
import { ListFilterModel } from "./list-filter/filter";
|
||||
import { SceneListFilterOptions } from "./list-filter/scenes";
|
||||
|
||||
export type QueuedScene = Pick<Scene, "id" | "title" | "paths">;
|
||||
|
||||
interface IQueryParameters {
|
||||
qsort?: string;
|
||||
qsortd?: string;
|
||||
|
|
|
|||
Loading…
Reference in a new issue