diff --git a/gqlgen.yml b/gqlgen.yml index d3b8fc67f..b949d44dc 100644 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -35,6 +35,8 @@ models: model: github.com/stashapp/stash/internal/api.BoolMap PluginConfigMap: model: github.com/stashapp/stash/internal/api.PluginConfigMap + File: + model: github.com/stashapp/stash/internal/api.File VideoFile: fields: # override float fields - #1572 diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index 51718aee3..1ca653403 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -6,6 +6,16 @@ type Query { findDefaultFilter(mode: FilterMode!): SavedFilter @deprecated(reason: "default filter now stored in UI config") + "Find a file by its id or path" + findFile(id: ID, path: String): BaseFile! + + "Queries for Files" + findFiles( + file_filter: FileFilterType + filter: FindFilterType + ids: [ID!] + ): FindFilesResultType! + "Find a scene by ID or Checksum" findScene(id: ID, checksum: String): Scene findSceneByHash(input: SceneHashInput!): Scene diff --git a/graphql/schema/types/file.graphql b/graphql/schema/types/file.graphql index 8dea777bd..c967c38f2 100644 --- a/graphql/schema/types/file.graphql +++ b/graphql/schema/types/file.graphql @@ -7,8 +7,11 @@ type Folder { id: ID! path: String! - parent_folder_id: ID - zip_file_id: ID + parent_folder_id: ID @deprecated(reason: "Use parent_folder instead") + zip_file_id: ID @deprecated(reason: "Use zip_file instead") + + parent_folder: Folder! + zip_file: BasicFile mod_time: Time! @@ -21,8 +24,32 @@ interface BaseFile { path: String! basename: String! - parent_folder_id: ID! - zip_file_id: ID + parent_folder_id: ID! @deprecated(reason: "Use parent_folder instead") + zip_file_id: ID @deprecated(reason: "Use zip_file instead") + + parent_folder: Folder! + zip_file: BasicFile + + mod_time: Time! + size: Int64! + + fingerprint(type: String!): String + fingerprints: [Fingerprint!]! + + created_at: Time! + updated_at: Time! +} + +type BasicFile implements BaseFile { + id: ID! + path: String! + basename: String! + + parent_folder_id: ID! @deprecated(reason: "Use parent_folder instead") + zip_file_id: ID @deprecated(reason: "Use zip_file instead") + + parent_folder: Folder! + zip_file: BasicFile mod_time: Time! size: Int64! @@ -39,8 +66,11 @@ type VideoFile implements BaseFile { path: String! basename: String! - parent_folder_id: ID! - zip_file_id: ID + parent_folder_id: ID! @deprecated(reason: "Use parent_folder instead") + zip_file_id: ID @deprecated(reason: "Use zip_file instead") + + parent_folder: Folder! + zip_file: BasicFile mod_time: Time! size: Int64! @@ -66,8 +96,11 @@ type ImageFile implements BaseFile { path: String! basename: String! - parent_folder_id: ID! - zip_file_id: ID + parent_folder_id: ID! @deprecated(reason: "Use parent_folder instead") + zip_file_id: ID @deprecated(reason: "Use zip_file instead") + + parent_folder: Folder! + zip_file: BasicFile mod_time: Time! size: Int64! @@ -75,6 +108,7 @@ type ImageFile implements BaseFile { fingerprint(type: String!): String fingerprints: [Fingerprint!]! + format: String! width: Int! height: Int! @@ -89,8 +123,11 @@ type GalleryFile implements BaseFile { path: String! basename: String! - parent_folder_id: ID! - zip_file_id: ID + parent_folder_id: ID! @deprecated(reason: "Use parent_folder instead") + zip_file_id: ID @deprecated(reason: "Use zip_file instead") + + parent_folder: Folder! + zip_file: BasicFile mod_time: Time! size: Int64! @@ -125,3 +162,17 @@ input FileSetFingerprintsInput { "only supplied fingerprint types will be modified" fingerprints: [SetFingerprintsInput!]! } + +type FindFilesResultType { + count: Int! + + "Total megapixels of any image files" + megapixels: Float! + "Total duration in seconds of any video files" + duration: Float! + + "Total file size in bytes" + size: Int! + + files: [BaseFile!]! +} diff --git a/graphql/schema/types/filters.graphql b/graphql/schema/types/filters.graphql index 14bb8680b..cab47172e 100644 --- a/graphql/schema/types/filters.graphql +++ b/graphql/schema/types/filters.graphql @@ -681,6 +681,77 @@ input ImageFilterType { tags_filter: TagFilterType } +input FileFilterType { + AND: FileFilterType + OR: FileFilterType + NOT: FileFilterType + + path: StringCriterionInput + basename: StringCriterionInput + dir: StringCriterionInput + + parent_folder: HierarchicalMultiCriterionInput + + "Filter by modification time" + mod_time: TimestampCriterionInput + + "Filter files that have an exact match available" + duplicated: PHashDuplicationCriterionInput + + "find files based on hash" + hashes: [FingerprintFilterInput!] + + video_file_filter: VideoFileFilterInput + image_file_filter: ImageFileFilterInput + + scene_count: IntCriterionInput + image_count: IntCriterionInput + gallery_count: IntCriterionInput + + "Filter by related scenes that meet this criteria" + scenes_filter: SceneFilterType + "Filter by related images that meet this criteria" + images_filter: ImageFilterType + "Filter by related galleries that meet this criteria" + galleries_filter: GalleryFilterType + + "Filter by creation time" + created_at: TimestampCriterionInput + "Filter by last update time" + updated_at: TimestampCriterionInput +} + +input VideoFileFilterInput { + resolution: ResolutionCriterionInput + orientation: OrientationCriterionInput + framerate: IntCriterionInput + bitrate: IntCriterionInput + format: StringCriterionInput + video_codec: StringCriterionInput + audio_codec: StringCriterionInput + + "in seconds" + duration: IntCriterionInput + + captions: StringCriterionInput + + interactive: Boolean + interactive_speed: IntCriterionInput +} + +input ImageFileFilterInput { + format: StringCriterionInput + resolution: ResolutionCriterionInput + orientation: OrientationCriterionInput +} + +input FingerprintFilterInput { + type: String! + value: String! + "Hamming distance - defaults to 0" + distance: Int +} + enum CriterionModifier { "=" EQUALS diff --git a/internal/api/fields.go b/internal/api/fields.go new file mode 100644 index 000000000..5f47ed06f --- /dev/null +++ b/internal/api/fields.go @@ -0,0 +1,23 @@ +package api + +import ( + "context" + + "github.com/99designs/gqlgen/graphql" +) + +type queryFields []string + +func collectQueryFields(ctx context.Context) queryFields { + fields := graphql.CollectAllFields(ctx) + return queryFields(fields) +} + +func (f queryFields) Has(field string) bool { + for _, v := range f { + if v == field { + return true + } + } + return false +} diff --git a/internal/api/loaders/dataloaders.go b/internal/api/loaders/dataloaders.go index 493c353d7..38f72b0a1 100644 --- a/internal/api/loaders/dataloaders.go +++ b/internal/api/loaders/dataloaders.go @@ -10,6 +10,7 @@ //go:generate go run github.com/vektah/dataloaden TagLoader int *github.com/stashapp/stash/pkg/models.Tag //go:generate go run github.com/vektah/dataloaden GroupLoader int *github.com/stashapp/stash/pkg/models.Group //go:generate go run github.com/vektah/dataloaden FileLoader github.com/stashapp/stash/pkg/models.FileID github.com/stashapp/stash/pkg/models.File +//go:generate go run github.com/vektah/dataloaden FolderLoader github.com/stashapp/stash/pkg/models.FolderID *github.com/stashapp/stash/pkg/models.Folder //go:generate go run github.com/vektah/dataloaden SceneFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID //go:generate go run github.com/vektah/dataloaden ImageFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID //go:generate go run github.com/vektah/dataloaden GalleryFileIDsLoader int []github.com/stashapp/stash/pkg/models.FileID @@ -62,6 +63,7 @@ type Loaders struct { TagByID *TagLoader GroupByID *GroupLoader FileByID *FileLoader + FolderByID *FolderLoader } type Middleware struct { @@ -117,6 +119,11 @@ func (m Middleware) Middleware(next http.Handler) http.Handler { maxBatch: maxBatch, fetch: m.fetchFiles(ctx), }, + FolderByID: &FolderLoader{ + wait: wait, + maxBatch: maxBatch, + fetch: m.fetchFolders(ctx), + }, SceneFiles: &SceneFileIDsLoader{ wait: wait, maxBatch: maxBatch, @@ -279,6 +286,17 @@ func (m Middleware) fetchFiles(ctx context.Context) func(keys []models.FileID) ( } } +func (m Middleware) fetchFolders(ctx context.Context) func(keys []models.FolderID) ([]*models.Folder, []error) { + return func(keys []models.FolderID) (ret []*models.Folder, errs []error) { + err := m.Repository.WithDB(ctx, func(ctx context.Context) error { + var err error + ret, err = m.Repository.Folder.FindMany(ctx, keys) + return err + }) + return ret, toErrorSlice(err) + } +} + func (m Middleware) fetchScenesFileIDs(ctx context.Context) func(keys []int) ([][]models.FileID, []error) { return func(keys []int) (ret [][]models.FileID, errs []error) { err := m.Repository.WithDB(ctx, func(ctx context.Context) error { diff --git a/internal/api/loaders/folderloader_gen.go b/internal/api/loaders/folderloader_gen.go new file mode 100644 index 000000000..ca2518b82 --- /dev/null +++ b/internal/api/loaders/folderloader_gen.go @@ -0,0 +1,224 @@ +// Code generated by github.com/vektah/dataloaden, DO NOT EDIT. + +package loaders + +import ( + "sync" + "time" + + "github.com/stashapp/stash/pkg/models" +) + +// FolderLoaderConfig captures the config to create a new FolderLoader +type FolderLoaderConfig struct { + // Fetch is a method that provides the data for the loader + Fetch func(keys []models.FolderID) ([]*models.Folder, []error) + + // Wait is how long wait before sending a batch + Wait time.Duration + + // MaxBatch will limit the maximum number of keys to send in one batch, 0 = not limit + MaxBatch int +} + +// NewFolderLoader creates a new FolderLoader given a fetch, wait, and maxBatch +func NewFolderLoader(config FolderLoaderConfig) *FolderLoader { + return &FolderLoader{ + fetch: config.Fetch, + wait: config.Wait, + maxBatch: config.MaxBatch, + } +} + +// FolderLoader batches and caches requests +type FolderLoader struct { + // this method provides the data for the loader + fetch func(keys []models.FolderID) ([]*models.Folder, []error) + + // how long to done before sending a batch + wait time.Duration + + // this will limit the maximum number of keys to send in one batch, 0 = no limit + maxBatch int + + // INTERNAL + + // lazily created cache + cache map[models.FolderID]*models.Folder + + // the current batch. keys will continue to be collected until timeout is hit, + // then everything will be sent to the fetch method and out to the listeners + batch *folderLoaderBatch + + // mutex to prevent races + mu sync.Mutex +} + +type folderLoaderBatch struct { + keys []models.FolderID + data []*models.Folder + error []error + closing bool + done chan struct{} +} + +// Load a Folder by key, batching and caching will be applied automatically +func (l *FolderLoader) Load(key models.FolderID) (*models.Folder, error) { + return l.LoadThunk(key)() +} + +// LoadThunk returns a function that when called will block waiting for a Folder. +// This method should be used if you want one goroutine to make requests to many +// different data loaders without blocking until the thunk is called. +func (l *FolderLoader) LoadThunk(key models.FolderID) func() (*models.Folder, error) { + l.mu.Lock() + if it, ok := l.cache[key]; ok { + l.mu.Unlock() + return func() (*models.Folder, error) { + return it, nil + } + } + if l.batch == nil { + l.batch = &folderLoaderBatch{done: make(chan struct{})} + } + batch := l.batch + pos := batch.keyIndex(l, key) + l.mu.Unlock() + + return func() (*models.Folder, error) { + <-batch.done + + var data *models.Folder + if pos < len(batch.data) { + data = batch.data[pos] + } + + var err error + // its convenient to be able to return a single error for everything + if len(batch.error) == 1 { + err = batch.error[0] + } else if batch.error != nil { + err = batch.error[pos] + } + + if err == nil { + l.mu.Lock() + l.unsafeSet(key, data) + l.mu.Unlock() + } + + return data, err + } +} + +// LoadAll fetches many keys at once. It will be broken into appropriate sized +// sub batches depending on how the loader is configured +func (l *FolderLoader) LoadAll(keys []models.FolderID) ([]*models.Folder, []error) { + results := make([]func() (*models.Folder, error), len(keys)) + + for i, key := range keys { + results[i] = l.LoadThunk(key) + } + + folders := make([]*models.Folder, len(keys)) + errors := make([]error, len(keys)) + for i, thunk := range results { + folders[i], errors[i] = thunk() + } + return folders, errors +} + +// LoadAllThunk returns a function that when called will block waiting for a Folders. +// This method should be used if you want one goroutine to make requests to many +// different data loaders without blocking until the thunk is called. +func (l *FolderLoader) LoadAllThunk(keys []models.FolderID) func() ([]*models.Folder, []error) { + results := make([]func() (*models.Folder, error), len(keys)) + for i, key := range keys { + results[i] = l.LoadThunk(key) + } + return func() ([]*models.Folder, []error) { + folders := make([]*models.Folder, len(keys)) + errors := make([]error, len(keys)) + for i, thunk := range results { + folders[i], errors[i] = thunk() + } + return folders, errors + } +} + +// Prime the cache with the provided key and value. If the key already exists, no change is made +// and false is returned. +// (To forcefully prime the cache, clear the key first with loader.clear(key).prime(key, value).) +func (l *FolderLoader) Prime(key models.FolderID, value *models.Folder) bool { + l.mu.Lock() + var found bool + if _, found = l.cache[key]; !found { + // make a copy when writing to the cache, its easy to pass a pointer in from a loop var + // and end up with the whole cache pointing to the same value. + cpy := *value + l.unsafeSet(key, &cpy) + } + l.mu.Unlock() + return !found +} + +// Clear the value at key from the cache, if it exists +func (l *FolderLoader) Clear(key models.FolderID) { + l.mu.Lock() + delete(l.cache, key) + l.mu.Unlock() +} + +func (l *FolderLoader) unsafeSet(key models.FolderID, value *models.Folder) { + if l.cache == nil { + l.cache = map[models.FolderID]*models.Folder{} + } + l.cache[key] = value +} + +// keyIndex will return the location of the key in the batch, if its not found +// it will add the key to the batch +func (b *folderLoaderBatch) keyIndex(l *FolderLoader, key models.FolderID) int { + for i, existingKey := range b.keys { + if key == existingKey { + return i + } + } + + pos := len(b.keys) + b.keys = append(b.keys, key) + if pos == 0 { + go b.startTimer(l) + } + + if l.maxBatch != 0 && pos >= l.maxBatch-1 { + if !b.closing { + b.closing = true + l.batch = nil + go b.end(l) + } + } + + return pos +} + +func (b *folderLoaderBatch) startTimer(l *FolderLoader) { + time.Sleep(l.wait) + l.mu.Lock() + + // we must have hit a batch limit and are already finalizing this batch + if b.closing { + l.mu.Unlock() + return + } + + l.batch = nil + l.mu.Unlock() + + b.end(l) +} + +func (b *folderLoaderBatch) end(l *FolderLoader) { + b.data, b.error = l.fetch(b.keys) + close(b.done) +} diff --git a/internal/api/models.go b/internal/api/models.go index d8f4dc63c..1c7346697 100644 --- a/internal/api/models.go +++ b/internal/api/models.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/sliceutil" ) type BaseFile interface { @@ -27,6 +28,29 @@ func convertVisualFile(f models.File) (VisualFile, error) { } } +func convertBaseFile(f models.File) BaseFile { + if f == nil { + return nil + } + + switch f := f.(type) { + case BaseFile: + return f + case *models.VideoFile: + return &VideoFile{VideoFile: f} + case *models.ImageFile: + return &ImageFile{ImageFile: f} + case *models.BaseFile: + return &BasicFile{BaseFile: f} + default: + panic("unknown file type") + } +} + +func convertBaseFiles(files []models.File) []BaseFile { + return sliceutil.Map(files, convertBaseFile) +} + type GalleryFile struct { *models.BaseFile } @@ -62,3 +86,15 @@ func (ImageFile) IsVisualFile() {} func (f *ImageFile) Fingerprints() []models.Fingerprint { return f.ImageFile.Fingerprints } + +type BasicFile struct { + *models.BaseFile +} + +func (BasicFile) IsBaseFile() {} + +func (BasicFile) IsVisualFile() {} + +func (f *BasicFile) Fingerprints() []models.Fingerprint { + return f.BaseFile.Fingerprints +} diff --git a/internal/api/resolver.go b/internal/api/resolver.go index f3097969d..061d0e1a9 100644 --- a/internal/api/resolver.go +++ b/internal/api/resolver.go @@ -95,6 +95,12 @@ func (r *Resolver) VideoFile() VideoFileResolver { func (r *Resolver) ImageFile() ImageFileResolver { return &imageFileResolver{r} } +func (r *Resolver) BasicFile() BasicFileResolver { + return &basicFileResolver{r} +} +func (r *Resolver) Folder() FolderResolver { + return &folderResolver{r} +} func (r *Resolver) SavedFilter() SavedFilterResolver { return &savedFilterResolver{r} } @@ -125,6 +131,8 @@ type tagResolver struct{ *Resolver } type galleryFileResolver struct{ *Resolver } type videoFileResolver struct{ *Resolver } type imageFileResolver struct{ *Resolver } +type basicFileResolver struct{ *Resolver } +type folderResolver struct{ *Resolver } type savedFilterResolver struct{ *Resolver } type pluginResolver struct{ *Resolver } type configResultResolver struct{ *Resolver } diff --git a/internal/api/resolver_model_file.go b/internal/api/resolver_model_file.go index 35013cfbd..4b9995311 100644 --- a/internal/api/resolver_model_file.go +++ b/internal/api/resolver_model_file.go @@ -1,30 +1,80 @@ package api -import "context" +import ( + "context" -func (r *galleryFileResolver) Fingerprint(ctx context.Context, obj *GalleryFile, type_ string) (*string, error) { - fp := obj.BaseFile.Fingerprints.For(type_) - if fp != nil { - v := fp.Value() - return &v, nil + "github.com/stashapp/stash/internal/api/loaders" + "github.com/stashapp/stash/pkg/models" +) + +func fingerprintResolver(fp models.Fingerprints, type_ string) (*string, error) { + fingerprint := fp.For(type_) + if fingerprint != nil { + value := fingerprint.Value() + return &value, nil } return nil, nil } +func (r *galleryFileResolver) Fingerprint(ctx context.Context, obj *GalleryFile, type_ string) (*string, error) { + return fingerprintResolver(obj.BaseFile.Fingerprints, type_) +} + func (r *imageFileResolver) Fingerprint(ctx context.Context, obj *ImageFile, type_ string) (*string, error) { - fp := obj.ImageFile.Fingerprints.For(type_) - if fp != nil { - v := fp.Value() - return &v, nil - } - return nil, nil + return fingerprintResolver(obj.ImageFile.Fingerprints, type_) } func (r *videoFileResolver) Fingerprint(ctx context.Context, obj *VideoFile, type_ string) (*string, error) { - fp := obj.VideoFile.Fingerprints.For(type_) - if fp != nil { - v := fp.Value() - return &v, nil - } - return nil, nil + return fingerprintResolver(obj.VideoFile.Fingerprints, type_) +} + +func (r *basicFileResolver) Fingerprint(ctx context.Context, obj *BasicFile, type_ string) (*string, error) { + return fingerprintResolver(obj.BaseFile.Fingerprints, type_) +} + +func (r *galleryFileResolver) ParentFolder(ctx context.Context, obj *GalleryFile) (*models.Folder, error) { + return loaders.From(ctx).FolderByID.Load(obj.ParentFolderID) +} + +func (r *imageFileResolver) ParentFolder(ctx context.Context, obj *ImageFile) (*models.Folder, error) { + return loaders.From(ctx).FolderByID.Load(obj.ParentFolderID) +} + +func (r *videoFileResolver) ParentFolder(ctx context.Context, obj *VideoFile) (*models.Folder, error) { + return loaders.From(ctx).FolderByID.Load(obj.ParentFolderID) +} + +func (r *basicFileResolver) ParentFolder(ctx context.Context, obj *BasicFile) (*models.Folder, error) { + return loaders.From(ctx).FolderByID.Load(obj.ParentFolderID) +} + +func zipFileResolver(ctx context.Context, zipFileID *models.FileID) (*BasicFile, error) { + if zipFileID == nil { + return nil, nil + } + + f, err := loaders.From(ctx).FileByID.Load(*zipFileID) + if err != nil { + return nil, err + } + + return &BasicFile{ + BaseFile: f.Base(), + }, nil +} + +func (r *galleryFileResolver) ZipFile(ctx context.Context, obj *GalleryFile) (*BasicFile, error) { + return zipFileResolver(ctx, obj.ZipFileID) +} + +func (r *imageFileResolver) ZipFile(ctx context.Context, obj *ImageFile) (*BasicFile, error) { + return zipFileResolver(ctx, obj.ZipFileID) +} + +func (r *videoFileResolver) ZipFile(ctx context.Context, obj *VideoFile) (*BasicFile, error) { + return zipFileResolver(ctx, obj.ZipFileID) +} + +func (r *basicFileResolver) ZipFile(ctx context.Context, obj *BasicFile) (*BasicFile, error) { + return zipFileResolver(ctx, obj.ZipFileID) } diff --git a/internal/api/resolver_model_folder.go b/internal/api/resolver_model_folder.go new file mode 100644 index 000000000..ee6bbfd05 --- /dev/null +++ b/internal/api/resolver_model_folder.go @@ -0,0 +1,20 @@ +package api + +import ( + "context" + + "github.com/stashapp/stash/internal/api/loaders" + "github.com/stashapp/stash/pkg/models" +) + +func (r *folderResolver) ParentFolder(ctx context.Context, obj *models.Folder) (*models.Folder, error) { + if obj.ParentFolderID == nil { + return nil, nil + } + + return loaders.From(ctx).FolderByID.Load(*obj.ParentFolderID) +} + +func (r *folderResolver) ZipFile(ctx context.Context, obj *models.Folder) (*BasicFile, error) { + return zipFileResolver(ctx, obj.ZipFileID) +} diff --git a/internal/api/resolver_query_find_file.go b/internal/api/resolver_query_find_file.go new file mode 100644 index 000000000..ae53a89b4 --- /dev/null +++ b/internal/api/resolver_query_find_file.go @@ -0,0 +1,120 @@ +package api + +import ( + "context" + "errors" + "strconv" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" +) + +func (r *queryResolver) FindFile(ctx context.Context, id *string, path *string) (BaseFile, error) { + var ret models.File + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + qb := r.repository.File + var err error + switch { + case id != nil: + idInt, err := strconv.Atoi(*id) + if err != nil { + return err + } + var files []models.File + files, err = qb.Find(ctx, models.FileID(idInt)) + if err != nil { + return err + } + if len(files) > 0 { + ret = files[0] + } + case path != nil: + ret, err = qb.FindByPath(ctx, *path) + if err == nil && ret == nil { + return errors.New("file not found") + } + default: + return errors.New("either id or path must be provided") + } + + return err + }); err != nil { + return nil, err + } + + return convertBaseFile(ret), nil +} + +func (r *queryResolver) FindFiles( + ctx context.Context, + fileFilter *models.FileFilterType, + filter *models.FindFilterType, + ids []string, +) (ret *FindFilesResultType, err error) { + var fileIDs []models.FileID + if len(ids) > 0 { + fileIDsInt, err := stringslice.StringSliceToIntSlice(ids) + if err != nil { + return nil, err + } + + fileIDs = models.FileIDsFromInts(fileIDsInt) + } + + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + var files []models.File + var err error + + fields := collectQueryFields(ctx) + result := &models.FileQueryResult{} + + if len(fileIDs) > 0 { + files, err = r.repository.File.Find(ctx, fileIDs...) + if err == nil { + result.Count = len(files) + for _, f := range files { + if asVideo, ok := f.(*models.VideoFile); ok { + result.TotalDuration += asVideo.Duration + } + if asImage, ok := f.(*models.ImageFile); ok { + result.Megapixels += asImage.Megapixels() + } + + result.TotalSize += f.Base().Size + } + } + } else { + result, err = r.repository.File.Query(ctx, models.FileQueryOptions{ + QueryOptions: models.QueryOptions{ + FindFilter: filter, + Count: fields.Has("count"), + }, + FileFilter: fileFilter, + TotalDuration: fields.Has("duration"), + Megapixels: fields.Has("megapixels"), + TotalSize: fields.Has("size"), + }) + if err == nil { + files, err = result.Resolve(ctx) + } + } + + if err != nil { + return err + } + + ret = &FindFilesResultType{ + Count: result.Count, + Files: convertBaseFiles(files), + Duration: result.TotalDuration, + Megapixels: result.Megapixels, + Size: int(result.TotalSize), + } + + return nil + }); err != nil { + return nil, err + } + + return ret, nil +} diff --git a/pkg/models/file.go b/pkg/models/file.go index e6ce41d1e..1b77af21a 100644 --- a/pkg/models/file.go +++ b/pkg/models/file.go @@ -9,15 +9,34 @@ import ( type FileQueryOptions struct { QueryOptions FileFilter *FileFilterType + + TotalDuration bool + Megapixels bool + TotalSize bool } type FileFilterType struct { - And *FileFilterType `json:"AND"` - Or *FileFilterType `json:"OR"` - Not *FileFilterType `json:"NOT"` + OperatorFilter[FileFilterType] // Filter by path Path *StringCriterionInput `json:"path"` + + Basename *StringCriterionInput `json:"basename"` + Dir *StringCriterionInput `json:"dir"` + ParentFolder *HierarchicalMultiCriterionInput `json:"parent_folder"` + ModTime *TimestampCriterionInput `json:"mod_time"` + Duplicated *PHashDuplicationCriterionInput `json:"duplicated"` + Hashes []*FingerprintFilterInput `json:"hashes"` + VideoFileFilter *VideoFileFilterInput `json:"video_file_filter"` + ImageFileFilter *ImageFileFilterInput `json:"image_file_filter"` + SceneCount *IntCriterionInput `json:"scene_count"` + ImageCount *IntCriterionInput `json:"image_count"` + GalleryCount *IntCriterionInput `json:"gallery_count"` + ScenesFilter *SceneFilterType `json:"scenes_filter"` + ImagesFilter *ImageFilterType `json:"images_filter"` + GalleriesFilter *GalleryFilterType `json:"galleries_filter"` + CreatedAt *TimestampCriterionInput `json:"created_at"` + UpdatedAt *TimestampCriterionInput `json:"updated_at"` } func PathsFileFilter(paths []string) *FileFilterType { @@ -53,10 +72,10 @@ func PathsFileFilter(paths []string) *FileFilterType { } type FileQueryResult struct { - // can't use QueryResult because id type is wrong - - IDs []FileID - Count int + QueryResult[FileID] + TotalDuration float64 + Megapixels float64 + TotalSize int64 getter FileGetter files []File diff --git a/pkg/models/filter.go b/pkg/models/filter.go index 2d25f6516..97d850a55 100644 --- a/pkg/models/filter.go +++ b/pkg/models/filter.go @@ -200,3 +200,31 @@ type CustomFieldCriterionInput struct { Value []any `json:"value"` Modifier CriterionModifier `json:"modifier"` } + +type FingerprintFilterInput struct { + Type string `json:"type"` + Value string `json:"value"` + // Hamming distance - defaults to 0 + Distance *int `json:"distance,omitempty"` +} + +type VideoFileFilterInput struct { + Format *StringCriterionInput `json:"format,omitempty"` + Resolution *ResolutionCriterionInput `json:"resolution,omitempty"` + Orientation *OrientationCriterionInput `json:"orientation,omitempty"` + Framerate *IntCriterionInput `json:"framerate,omitempty"` + Bitrate *IntCriterionInput `json:"bitrate,omitempty"` + VideoCodec *StringCriterionInput `json:"video_codec,omitempty"` + AudioCodec *StringCriterionInput `json:"audio_codec,omitempty"` + // in seconds + Duration *IntCriterionInput `json:"duration,omitempty"` + Captions *StringCriterionInput `json:"captions,omitempty"` + Interactive *bool `json:"interactive,omitempty"` + InteractiveSpeed *IntCriterionInput `json:"interactive_speed,omitempty"` +} + +type ImageFileFilterInput struct { + Format *StringCriterionInput `json:"format,omitempty"` + Resolution *ResolutionCriterionInput `json:"resolution,omitempty"` + Orientation *OrientationCriterionInput `json:"orientation,omitempty"` +} diff --git a/pkg/models/image.go b/pkg/models/image.go index 370315159..9d2c6f016 100644 --- a/pkg/models/image.go +++ b/pkg/models/image.go @@ -106,7 +106,7 @@ type ImageQueryOptions struct { } type ImageQueryResult struct { - QueryResult + QueryResult[int] Megapixels float64 TotalSize float64 diff --git a/pkg/models/mocks/FolderReaderWriter.go b/pkg/models/mocks/FolderReaderWriter.go index 968bed4ad..020764942 100644 --- a/pkg/models/mocks/FolderReaderWriter.go +++ b/pkg/models/mocks/FolderReaderWriter.go @@ -178,6 +178,29 @@ func (_m *FolderReaderWriter) FindByZipFileID(ctx context.Context, zipFileID mod return r0, r1 } +// FindMany provides a mock function with given fields: ctx, id +func (_m *FolderReaderWriter) FindMany(ctx context.Context, id []models.FolderID) ([]*models.Folder, error) { + ret := _m.Called(ctx, id) + + var r0 []*models.Folder + if rf, ok := ret.Get(0).(func(context.Context, []models.FolderID) []*models.Folder); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Folder) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, []models.FolderID) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Update provides a mock function with given fields: ctx, f func (_m *FolderReaderWriter) Update(ctx context.Context, f *models.Folder) error { ret := _m.Called(ctx, f) diff --git a/pkg/models/model_file.go b/pkg/models/model_file.go index e9df57990..f6b8bdc51 100644 --- a/pkg/models/model_file.go +++ b/pkg/models/model_file.go @@ -79,6 +79,14 @@ func (i FileID) MarshalGQL(w io.Writer) { fmt.Fprint(w, strconv.Quote(i.String())) } +func FileIDsFromInts(ids []int) []FileID { + ret := make([]FileID, len(ids)) + for i, id := range ids { + ret[i] = FileID(id) + } + return ret +} + // DirEntry represents a file or directory in the file system. type DirEntry struct { ZipFileID *FileID `json:"zip_file_id"` @@ -252,6 +260,10 @@ func (f ImageFile) GetHeight() int { return f.Height } +func (f ImageFile) Megapixels() float64 { + return float64(f.Width*f.Height) / 1e6 +} + func (f ImageFile) GetFormat() string { return f.Format } diff --git a/pkg/models/query.go b/pkg/models/query.go index 1b2d347b9..a6e15bc4e 100644 --- a/pkg/models/query.go +++ b/pkg/models/query.go @@ -5,7 +5,7 @@ type QueryOptions struct { Count bool } -type QueryResult struct { - IDs []int +type QueryResult[T comparable] struct { + IDs []T Count int } diff --git a/pkg/models/repository_folder.go b/pkg/models/repository_folder.go index c3f82f529..20c155ead 100644 --- a/pkg/models/repository_folder.go +++ b/pkg/models/repository_folder.go @@ -5,6 +5,7 @@ import "context" // FolderGetter provides methods to get folders by ID. type FolderGetter interface { Find(ctx context.Context, id FolderID) (*Folder, error) + FindMany(ctx context.Context, id []FolderID) ([]*Folder, error) } // FolderFinder provides methods to find folders. diff --git a/pkg/models/scene.go b/pkg/models/scene.go index c7be343d9..9f28d40ba 100644 --- a/pkg/models/scene.go +++ b/pkg/models/scene.go @@ -126,7 +126,7 @@ type SceneQueryOptions struct { } type SceneQueryResult struct { - QueryResult + QueryResult[int] TotalDuration float64 TotalSize float64 diff --git a/pkg/sqlite/batch.go b/pkg/sqlite/batch.go index 71ad5d354..a59438835 100644 --- a/pkg/sqlite/batch.go +++ b/pkg/sqlite/batch.go @@ -3,7 +3,7 @@ package sqlite const defaultBatchSize = 1000 // batchExec executes the provided function in batches of the provided size. -func batchExec(ids []int, batchSize int, fn func(batch []int) error) error { +func batchExec[T any](ids []T, batchSize int, fn func(batch []T) error) error { for i := 0; i < len(ids); i += batchSize { end := i + batchSize if end > len(ids) { diff --git a/pkg/sqlite/criterion_handlers.go b/pkg/sqlite/criterion_handlers.go index 55ff31fca..82b9cfc65 100644 --- a/pkg/sqlite/criterion_handlers.go +++ b/pkg/sqlite/criterion_handlers.go @@ -70,6 +70,17 @@ func stringCriterionHandler(c *models.StringCriterionInput, column string) crite } } +func joinedStringCriterionHandler(c *models.StringCriterionInput, column string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc { + return func(ctx context.Context, f *filterBuilder) { + if c != nil { + if addJoinFn != nil { + addJoinFn(f) + } + stringCriterionHandler(c, column)(ctx, f) + } + } +} + func enumCriterionHandler(modifier models.CriterionModifier, values []string, column string) criterionHandlerFunc { return func(ctx context.Context, f *filterBuilder) { if modifier.IsValid() { diff --git a/pkg/sqlite/file.go b/pkg/sqlite/file.go index 6bf6e32b5..ea2084c2c 100644 --- a/pkg/sqlite/file.go +++ b/pkg/sqlite/file.go @@ -275,6 +275,43 @@ func (r fileQueryRows) resolve() []models.File { return ret } +type fileRepositoryType struct { + repository + scenes joinRepository + images joinRepository + galleries joinRepository +} + +var ( + fileRepository = fileRepositoryType{ + repository: repository{ + tableName: sceneTable, + idColumn: idColumn, + }, + scenes: joinRepository{ + repository: repository{ + tableName: scenesFilesTable, + idColumn: fileIDColumn, + }, + fkColumn: sceneIDColumn, + }, + images: joinRepository{ + repository: repository{ + tableName: imagesFilesTable, + idColumn: fileIDColumn, + }, + fkColumn: imageIDColumn, + }, + galleries: joinRepository{ + repository: repository{ + tableName: galleriesFilesTable, + idColumn: fileIDColumn, + }, + fkColumn: galleryIDColumn, + }, + } +) + type FileStore struct { repository @@ -830,9 +867,11 @@ func (qb *FileStore) makeFilter(ctx context.Context, fileFilter *models.FileFilt query.not(qb.makeFilter(ctx, fileFilter.Not)) } - query.handleCriterion(ctx, pathCriterionHandler(fileFilter.Path, "folders.path", "files.basename", nil)) + filter := filterBuilderFromHandler(ctx, &fileFilterHandler{ + fileFilter: fileFilter, + }) - return query + return filter } func (qb *FileStore) Query(ctx context.Context, options models.FileQueryOptions) (*models.FileQueryResult, error) { @@ -890,7 +929,7 @@ func (qb *FileStore) Query(ctx context.Context, options models.FileQueryOptions) } func (qb *FileStore) queryGroupedFields(ctx context.Context, options models.FileQueryOptions, query queryBuilder) (*models.FileQueryResult, error) { - if !options.Count { + if !options.Count && !options.TotalDuration && !options.Megapixels && !options.TotalSize { // nothing to do - return empty result return models.NewFileQueryResult(qb), nil } @@ -898,14 +937,43 @@ func (qb *FileStore) queryGroupedFields(ctx context.Context, options models.File aggregateQuery := qb.newQuery() if options.Count { - aggregateQuery.addColumn("COUNT(temp.id) as total") + aggregateQuery.addColumn("COUNT(DISTINCT temp.id) as total") + } + + if options.TotalDuration { + query.addJoins( + join{ + table: videoFileTable, + onClause: "files.id = video_files.file_id", + }, + ) + query.addColumn("COALESCE(video_files.duration, 0) as duration") + aggregateQuery.addColumn("COALESCE(SUM(temp.duration), 0) as duration") + } + if options.Megapixels { + query.addJoins( + join{ + table: imageFileTable, + onClause: "files.id = image_files.file_id", + }, + ) + query.addColumn("COALESCE(image_files.width, 0) * COALESCE(image_files.height, 0) as megapixels") + aggregateQuery.addColumn("COALESCE(SUM(temp.megapixels), 0) / 1000000 as megapixels") + } + + if options.TotalSize { + query.addColumn("COALESCE(files.size, 0) as size") + aggregateQuery.addColumn("COALESCE(SUM(temp.size), 0) as size") } const includeSortPagination = false aggregateQuery.from = fmt.Sprintf("(%s) as temp", query.toSQL(includeSortPagination)) out := struct { - Total int + Total int + Duration float64 + Megapixels float64 + Size int64 }{} if err := qb.repository.queryStruct(ctx, aggregateQuery.toSQL(includeSortPagination), query.args, &out); err != nil { return nil, err @@ -913,6 +981,9 @@ func (qb *FileStore) queryGroupedFields(ctx context.Context, options models.File ret := models.NewFileQueryResult(qb) ret.Count = out.Total + ret.Megapixels = out.Megapixels + ret.TotalDuration = out.Duration + ret.TotalSize = out.Size return ret, nil } diff --git a/pkg/sqlite/file_filter.go b/pkg/sqlite/file_filter.go new file mode 100644 index 000000000..b115fee35 --- /dev/null +++ b/pkg/sqlite/file_filter.go @@ -0,0 +1,302 @@ +package sqlite + +import ( + "context" + "fmt" + + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" +) + +type fileFilterHandler struct { + fileFilter *models.FileFilterType +} + +func (qb *fileFilterHandler) validate() error { + fileFilter := qb.fileFilter + if fileFilter == nil { + return nil + } + + if err := validateFilterCombination(fileFilter.OperatorFilter); err != nil { + return err + } + + if subFilter := fileFilter.SubFilter(); subFilter != nil { + sqb := &fileFilterHandler{fileFilter: subFilter} + if err := sqb.validate(); err != nil { + return err + } + } + + return nil +} + +func (qb *fileFilterHandler) handle(ctx context.Context, f *filterBuilder) { + fileFilter := qb.fileFilter + if fileFilter == nil { + return + } + + if err := qb.validate(); err != nil { + f.setError(err) + return + } + + sf := fileFilter.SubFilter() + if sf != nil { + sub := &fileFilterHandler{sf} + handleSubFilter(ctx, sub, f, fileFilter.OperatorFilter) + } + + f.handleCriterion(ctx, qb.criterionHandler()) +} + +func (qb *fileFilterHandler) criterionHandler() criterionHandler { + fileFilter := qb.fileFilter + return compoundHandler{ + &videoFileFilterHandler{ + filter: fileFilter.VideoFileFilter, + }, + &imageFileFilterHandler{ + filter: fileFilter.ImageFileFilter, + }, + + pathCriterionHandler(fileFilter.Path, "folders.path", "files.basename", nil), + stringCriterionHandler(fileFilter.Basename, "files.basename"), + stringCriterionHandler(fileFilter.Dir, "folders.path"), + ×tampCriterionHandler{fileFilter.ModTime, "files.mod_time", nil}, + + qb.parentFolderCriterionHandler(fileFilter.ParentFolder), + + qb.sceneCountCriterionHandler(fileFilter.SceneCount), + qb.imageCountCriterionHandler(fileFilter.ImageCount), + qb.galleryCountCriterionHandler(fileFilter.GalleryCount), + + qb.hashesCriterionHandler(fileFilter.Hashes), + + qb.phashDuplicatedCriterionHandler(fileFilter.Duplicated), + ×tampCriterionHandler{fileFilter.CreatedAt, "files.created_at", nil}, + ×tampCriterionHandler{fileFilter.UpdatedAt, "files.updated_at", nil}, + + &relatedFilterHandler{ + relatedIDCol: "scenes_files.scene_id", + relatedRepo: sceneRepository.repository, + relatedHandler: &sceneFilterHandler{fileFilter.ScenesFilter}, + joinFn: func(f *filterBuilder) { + fileRepository.scenes.innerJoin(f, "", "files.id") + }, + }, + &relatedFilterHandler{ + relatedIDCol: "images_files.image_id", + relatedRepo: imageRepository.repository, + relatedHandler: &imageFilterHandler{fileFilter.ImagesFilter}, + joinFn: func(f *filterBuilder) { + fileRepository.images.innerJoin(f, "", "files.id") + }, + }, + &relatedFilterHandler{ + relatedIDCol: "galleries_files.gallery_id", + relatedRepo: galleryRepository.repository, + relatedHandler: &galleryFilterHandler{fileFilter.GalleriesFilter}, + joinFn: func(f *filterBuilder) { + fileRepository.galleries.innerJoin(f, "", "files.id") + }, + }, + } +} + +func (qb *fileFilterHandler) parentFolderCriterionHandler(folder *models.HierarchicalMultiCriterionInput) criterionHandlerFunc { + return func(ctx context.Context, f *filterBuilder) { + if folder == nil { + return + } + + folderCopy := *folder + switch folderCopy.Modifier { + case models.CriterionModifierEquals: + folderCopy.Modifier = models.CriterionModifierIncludesAll + case models.CriterionModifierNotEquals: + folderCopy.Modifier = models.CriterionModifierExcludes + } + + hh := hierarchicalMultiCriterionHandlerBuilder{ + primaryTable: fileTable, + foreignTable: folderTable, + foreignFK: "parent_folder_id", + parentFK: "parent_folder_id", + } + + hh.handler(&folderCopy)(ctx, f) + } +} + +func (qb *fileFilterHandler) sceneCountCriterionHandler(c *models.IntCriterionInput) criterionHandlerFunc { + h := countCriterionHandlerBuilder{ + primaryTable: fileTable, + joinTable: scenesFilesTable, + primaryFK: fileIDColumn, + } + + return h.handler(c) +} + +func (qb *fileFilterHandler) imageCountCriterionHandler(c *models.IntCriterionInput) criterionHandlerFunc { + h := countCriterionHandlerBuilder{ + primaryTable: fileTable, + joinTable: imagesFilesTable, + primaryFK: fileIDColumn, + } + + return h.handler(c) +} + +func (qb *fileFilterHandler) galleryCountCriterionHandler(c *models.IntCriterionInput) criterionHandlerFunc { + h := countCriterionHandlerBuilder{ + primaryTable: fileTable, + joinTable: galleriesFilesTable, + primaryFK: fileIDColumn, + } + + return h.handler(c) +} + +func (qb *fileFilterHandler) phashDuplicatedCriterionHandler(duplicatedFilter *models.PHashDuplicationCriterionInput) criterionHandlerFunc { + return func(ctx context.Context, f *filterBuilder) { + // TODO: Wishlist item: Implement Distance matching + if duplicatedFilter != nil { + var v string + if *duplicatedFilter.Duplicated { + v = ">" + } else { + v = "=" + } + + f.addInnerJoin("(SELECT file_id FROM files_fingerprints INNER JOIN (SELECT fingerprint FROM files_fingerprints WHERE type = 'phash' GROUP BY fingerprint HAVING COUNT (fingerprint) "+v+" 1) dupes on files_fingerprints.fingerprint = dupes.fingerprint)", "scph", "files.id = scph.file_id") + } + } +} + +func (qb *fileFilterHandler) hashesCriterionHandler(hashes []*models.FingerprintFilterInput) criterionHandlerFunc { + return func(ctx context.Context, f *filterBuilder) { + // TODO - this won't work for AND/OR combinations + for i, hash := range hashes { + t := fmt.Sprintf("file_fingerprints_%d", i) + f.addLeftJoin(fingerprintTable, t, fmt.Sprintf("files.id = %s.file_id AND %s.type = ?", t, t), hash.Type) + + value, _ := utils.StringToPhash(hash.Value) + distance := 0 + if hash.Distance != nil { + distance = *hash.Distance + } + + if distance > 0 { + // needed to avoid a type mismatch + f.addWhere(fmt.Sprintf("typeof(%s.fingerprint) = 'integer'", t)) + f.addWhere(fmt.Sprintf("phash_distance(%s.fingerprint, ?) < ?", t), value, distance) + } else { + // use the default handler + intCriterionHandler(&models.IntCriterionInput{ + Value: int(value), + Modifier: models.CriterionModifierEquals, + }, t+".fingerprint", nil)(ctx, f) + } + } + } +} + +type videoFileFilterHandler struct { + filter *models.VideoFileFilterInput +} + +func (qb *videoFileFilterHandler) handle(ctx context.Context, f *filterBuilder) { + videoFileFilter := qb.filter + if videoFileFilter == nil { + return + } + f.handleCriterion(ctx, qb.criterionHandler()) +} + +func (qb *videoFileFilterHandler) criterionHandler() criterionHandler { + videoFileFilter := qb.filter + return compoundHandler{ + joinedStringCriterionHandler(videoFileFilter.Format, "video_files.format", qb.addVideoFilesTable), + floatIntCriterionHandler(videoFileFilter.Duration, "video_files.duration", qb.addVideoFilesTable), + resolutionCriterionHandler(videoFileFilter.Resolution, "video_files.height", "video_files.width", qb.addVideoFilesTable), + orientationCriterionHandler(videoFileFilter.Orientation, "video_files.height", "video_files.width", qb.addVideoFilesTable), + floatIntCriterionHandler(videoFileFilter.Framerate, "ROUND(video_files.frame_rate)", qb.addVideoFilesTable), + intCriterionHandler(videoFileFilter.Bitrate, "video_files.bit_rate", qb.addVideoFilesTable), + qb.codecCriterionHandler(videoFileFilter.VideoCodec, "video_files.video_codec", qb.addVideoFilesTable), + qb.codecCriterionHandler(videoFileFilter.AudioCodec, "video_files.audio_codec", qb.addVideoFilesTable), + + boolCriterionHandler(videoFileFilter.Interactive, "video_files.interactive", qb.addVideoFilesTable), + intCriterionHandler(videoFileFilter.InteractiveSpeed, "video_files.interactive_speed", qb.addVideoFilesTable), + + qb.captionCriterionHandler(videoFileFilter.Captions), + } +} + +func (qb *videoFileFilterHandler) addVideoFilesTable(f *filterBuilder) { + f.addLeftJoin(videoFileTable, "", "video_files.file_id = files.id") +} + +func (qb *videoFileFilterHandler) codecCriterionHandler(codec *models.StringCriterionInput, codecColumn string, addJoinFn func(f *filterBuilder)) criterionHandlerFunc { + return func(ctx context.Context, f *filterBuilder) { + if codec != nil { + if addJoinFn != nil { + addJoinFn(f) + } + + stringCriterionHandler(codec, codecColumn)(ctx, f) + } + } +} + +func (qb *videoFileFilterHandler) captionCriterionHandler(captions *models.StringCriterionInput) criterionHandlerFunc { + h := stringListCriterionHandlerBuilder{ + primaryTable: sceneTable, + primaryFK: sceneIDColumn, + joinTable: videoCaptionsTable, + stringColumn: captionCodeColumn, + addJoinTable: func(f *filterBuilder) { + f.addLeftJoin(videoCaptionsTable, "", "video_captions.file_id = files.id") + }, + excludeHandler: func(f *filterBuilder, criterion *models.StringCriterionInput) { + excludeClause := `files.id NOT IN ( + SELECT files.id from files + INNER JOIN video_captions on video_captions.file_id = files.id + WHERE video_captions.language_code LIKE ? + )` + f.addWhere(excludeClause, criterion.Value) + + // TODO - should we also exclude null values? + }, + } + + return h.handler(captions) +} + +type imageFileFilterHandler struct { + filter *models.ImageFileFilterInput +} + +func (qb *imageFileFilterHandler) handle(ctx context.Context, f *filterBuilder) { + ff := qb.filter + if ff == nil { + return + } + f.handleCriterion(ctx, qb.criterionHandler()) +} + +func (qb *imageFileFilterHandler) criterionHandler() criterionHandler { + ff := qb.filter + return compoundHandler{ + joinedStringCriterionHandler(ff.Format, "image_files.format", qb.addImageFilesTable), + resolutionCriterionHandler(ff.Resolution, "image_files.height", "image_files.width", qb.addImageFilesTable), + orientationCriterionHandler(ff.Orientation, "image_files.height", "image_files.width", qb.addImageFilesTable), + } +} + +func (qb *imageFileFilterHandler) addImageFilesTable(f *filterBuilder) { + f.addLeftJoin(imageFileTable, "", "image_files.file_id = files.id") +} diff --git a/pkg/sqlite/file_filter_test.go b/pkg/sqlite/file_filter_test.go new file mode 100644 index 000000000..7bc6f3e6b --- /dev/null +++ b/pkg/sqlite/file_filter_test.go @@ -0,0 +1,101 @@ +//go:build integration +// +build integration + +package sqlite_test + +import ( + "context" + "strconv" + "testing" + + "github.com/stashapp/stash/pkg/models" + "github.com/stretchr/testify/assert" +) + +func TestFileQuery(t *testing.T) { + tests := []struct { + name string + findFilter *models.FindFilterType + filter *models.FileFilterType + includeIdxs []int + includeIDs []int + excludeIdxs []int + wantErr bool + }{ + { + name: "path", + filter: &models.FileFilterType{ + Path: &models.StringCriterionInput{ + Value: getPrefixedStringValue("file", fileIdxStartVideoFiles, "basename"), + Modifier: models.CriterionModifierIncludes, + }, + }, + includeIdxs: []int{fileIdxStartVideoFiles}, + excludeIdxs: []int{fileIdxStartImageFiles}, + }, + { + name: "basename", + filter: &models.FileFilterType{ + Basename: &models.StringCriterionInput{ + Value: getPrefixedStringValue("file", fileIdxStartVideoFiles, "basename"), + Modifier: models.CriterionModifierIncludes, + }, + }, + includeIdxs: []int{fileIdxStartVideoFiles}, + excludeIdxs: []int{fileIdxStartImageFiles}, + }, + { + name: "dir", + filter: &models.FileFilterType{ + Path: &models.StringCriterionInput{ + Value: folderPaths[folderIdxWithSceneFiles], + Modifier: models.CriterionModifierIncludes, + }, + }, + includeIDs: []int{int(sceneFileIDs[sceneIdxWithGroup])}, + excludeIdxs: []int{fileIdxStartImageFiles}, + }, + { + name: "parent folder", + filter: &models.FileFilterType{ + ParentFolder: &models.HierarchicalMultiCriterionInput{ + Value: []string{ + strconv.Itoa(int(folderIDs[folderIdxWithSceneFiles])), + }, + Modifier: models.CriterionModifierIncludes, + }, + }, + includeIDs: []int{int(sceneFileIDs[sceneIdxWithGroup])}, + excludeIdxs: []int{fileIdxStartImageFiles}, + }, + // TODO - add more tests for other file filters + } + + for _, tt := range tests { + runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) { + assert := assert.New(t) + + results, err := db.File.Query(ctx, models.FileQueryOptions{ + FileFilter: tt.filter, + QueryOptions: models.QueryOptions{ + FindFilter: tt.findFilter, + }, + }) + if (err != nil) != tt.wantErr { + t.Errorf("SceneStore.Query() error = %v, wantErr %v", err, tt.wantErr) + return + } + + include := indexesToIDs(sceneIDs, tt.includeIdxs) + include = append(include, tt.includeIDs...) + exclude := indexesToIDs(sceneIDs, tt.excludeIdxs) + + for _, i := range include { + assert.Contains(results.IDs, models.FileID(i)) + } + for _, e := range exclude { + assert.NotContains(results.IDs, models.FileID(e)) + } + }) + } +} diff --git a/pkg/sqlite/folder.go b/pkg/sqlite/folder.go index 4cf632d49..f90a578bd 100644 --- a/pkg/sqlite/folder.go +++ b/pkg/sqlite/folder.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "path/filepath" + "slices" "github.com/doug-martin/goqu/v9" "github.com/doug-martin/goqu/v9/exp" @@ -225,6 +226,52 @@ func (qb *FolderStore) Find(ctx context.Context, id models.FolderID) (*models.Fo return ret, nil } +// FindByIDs finds multiple folders by their IDs. +// No check is made to see if the folders exist, and the order of the returned folders +// is not guaranteed to be the same as the order of the input IDs. +func (qb *FolderStore) FindByIDs(ctx context.Context, ids []models.FolderID) ([]*models.Folder, error) { + folders := make([]*models.Folder, 0, len(ids)) + + table := qb.table() + if err := batchExec(ids, defaultBatchSize, func(batch []models.FolderID) error { + q := qb.selectDataset().Prepared(true).Where(table.Col(idColumn).In(batch)) + unsorted, err := qb.getMany(ctx, q) + if err != nil { + return err + } + + folders = append(folders, unsorted...) + + return nil + }); err != nil { + return nil, err + } + + return folders, nil +} + +func (qb *FolderStore) FindMany(ctx context.Context, ids []models.FolderID) ([]*models.Folder, error) { + folders := make([]*models.Folder, len(ids)) + + unsorted, err := qb.FindByIDs(ctx, ids) + if err != nil { + return nil, err + } + + for _, s := range unsorted { + i := slices.Index(ids, s.ID) + folders[i] = s + } + + for i := range folders { + if folders[i] == nil { + return nil, fmt.Errorf("folder with id %d not found", ids[i]) + } + } + + return folders, nil +} + func (qb *FolderStore) FindByPath(ctx context.Context, p string) (*models.Folder, error) { q := qb.selectDataset().Prepared(true).Where(qb.table().Col("path").Eq(p))