Filter tag by hierarchy (#1746)

* Add API support for filtering tags by parent / children
* Add parent & child tags filters for tags to UI
* Add API support for filtering tags by parent / child count
* Add parent & child count filters for tags to UI
* Update db generator
* Add missing build tag
* Add unit tests

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>
This commit is contained in:
gitgiggety 2021-10-01 03:50:06 +02:00 committed by GitHub
parent df2c9e9754
commit dabf5acefe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 515 additions and 24 deletions

View file

@ -275,6 +275,18 @@ input TagFilterType {
"""Filter by number of markers with this tag"""
marker_count: IntCriterionInput
"""Filter by parent tags"""
parents: HierarchicalMultiCriterionInput
"""Filter by child tags"""
children: HierarchicalMultiCriterionInput
"""Filter by number of parent tags the tag has"""
parent_count: IntCriterionInput
"""Filter by number f child tags the tag has"""
child_count: IntCriterionInput
}
input ImageFilterType {

View file

@ -151,6 +151,11 @@ const (
tagIdxWithGallery
tagIdx1WithGallery
tagIdx2WithGallery
tagIdxWithChildTag
tagIdxWithParentTag
tagIdxWithGrandChild
tagIdxWithParentAndChild
tagIdxWithGrandParent
// new indexes above
// tags with dup names start from the end
tagIdx1WithDupName
@ -345,6 +350,14 @@ var (
}
)
var (
tagParentLinks = [][2]int{
{tagIdxWithChildTag, tagIdxWithParentTag},
{tagIdxWithGrandChild, tagIdxWithParentAndChild},
{tagIdxWithParentAndChild, tagIdxWithGrandParent},
}
)
func TestMain(m *testing.M) {
ret := runTests(m)
os.Exit(ret)
@ -499,6 +512,10 @@ func populateDB() error {
return fmt.Errorf("error linking gallery studios: %s", err.Error())
}
if err := linkTagsParent(r.Tag()); err != nil {
return fmt.Errorf("error linking tags parent: %s", err.Error())
}
if err := createMarker(r.SceneMarker(), sceneIdxWithMarker, tagIdxWithPrimaryMarker, []int{tagIdxWithMarker}); err != nil {
return fmt.Errorf("error creating scene marker: %s", err.Error())
}
@ -865,6 +882,22 @@ func getTagPerformerCount(id int) int {
return 0
}
func getTagParentCount(id int) int {
if id == tagIDs[tagIdxWithParentTag] || id == tagIDs[tagIdxWithGrandParent] || id == tagIDs[tagIdxWithParentAndChild] {
return 1
}
return 0
}
func getTagChildCount(id int) int {
if id == tagIDs[tagIdxWithChildTag] || id == tagIDs[tagIdxWithGrandChild] || id == tagIDs[tagIdxWithParentAndChild] {
return 1
}
return 0
}
//createTags creates n tags with plain Name and o tags with camel cased NaMe included
func createTags(tqb models.TagReaderWriter, n int, o int) error {
const namePlain = "Name"
@ -1231,6 +1264,25 @@ func linkStudiosParent(qb models.StudioWriter) error {
})
}
func linkTagsParent(qb models.TagReaderWriter) error {
return doLinks(tagParentLinks, func(parentIndex, childIndex int) error {
tagID := tagIDs[childIndex]
parentTags, err := qb.FindByChildTagID(tagID)
if err != nil {
return err
}
var parentIDs []int
for _, parentTag := range parentTags {
parentIDs = append(parentIDs, parentTag.ID)
}
parentIDs = append(parentIDs, tagIDs[parentIndex])
return qb.UpdateParentTags(tagID, parentIDs)
})
}
func addTagImage(qb models.TagWriter, tagIndex int) error {
return qb.UpdateImage(tagIDs[tagIndex], models.DefaultTagImage)
}

View file

@ -302,6 +302,10 @@ func (qb *tagQueryBuilder) makeFilter(tagFilter *models.TagFilterType) *filterBu
query.handleCriterion(tagGalleryCountCriterionHandler(qb, tagFilter.GalleryCount))
query.handleCriterion(tagPerformerCountCriterionHandler(qb, tagFilter.PerformerCount))
query.handleCriterion(tagMarkerCountCriterionHandler(qb, tagFilter.MarkerCount))
query.handleCriterion(tagParentsCriterionHandler(qb, tagFilter.Parents))
query.handleCriterion(tagChildrenCriterionHandler(qb, tagFilter.Children))
query.handleCriterion(tagParentCountCriterionHandler(qb, tagFilter.ParentCount))
query.handleCriterion(tagChildCountCriterionHandler(qb, tagFilter.ChildCount))
return query
}
@ -433,6 +437,94 @@ func tagMarkerCountCriterionHandler(qb *tagQueryBuilder, markerCount *models.Int
}
}
func tagParentsCriterionHandler(qb *tagQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc {
return func(f *filterBuilder) {
if tags != nil && len(tags.Value) > 0 {
var args []interface{}
for _, val := range tags.Value {
args = append(args, val)
}
depthVal := 0
if tags.Depth != nil {
depthVal = *tags.Depth
}
var depthCondition string
if depthVal != -1 {
depthCondition = fmt.Sprintf("WHERE depth < %d", depthVal)
}
query := `parents AS (
SELECT parent_id AS root_id, child_id AS item_id, 0 AS depth FROM tags_relations WHERE parent_id IN` + getInBinding(len(tags.Value)) + `
UNION
SELECT root_id, child_id, depth + 1 FROM tags_relations INNER JOIN parents ON item_id = parent_id ` + depthCondition + `
)`
f.addRecursiveWith(query, args...)
f.addJoin("parents", "", "parents.item_id = tags.id")
addHierarchicalConditionClauses(f, tags, "parents", "root_id")
}
}
}
func tagChildrenCriterionHandler(qb *tagQueryBuilder, tags *models.HierarchicalMultiCriterionInput) criterionHandlerFunc {
return func(f *filterBuilder) {
if tags != nil && len(tags.Value) > 0 {
var args []interface{}
for _, val := range tags.Value {
args = append(args, val)
}
depthVal := 0
if tags.Depth != nil {
depthVal = *tags.Depth
}
var depthCondition string
if depthVal != -1 {
depthCondition = fmt.Sprintf("WHERE depth < %d", depthVal)
}
query := `children AS (
SELECT child_id AS root_id, parent_id AS item_id, 0 AS depth FROM tags_relations WHERE child_id IN` + getInBinding(len(tags.Value)) + `
UNION
SELECT root_id, parent_id, depth + 1 FROM tags_relations INNER JOIN children ON item_id = child_id ` + depthCondition + `
)`
f.addRecursiveWith(query, args...)
f.addJoin("children", "", "children.item_id = tags.id")
addHierarchicalConditionClauses(f, tags, "children", "root_id")
}
}
}
func tagParentCountCriterionHandler(qb *tagQueryBuilder, parentCount *models.IntCriterionInput) criterionHandlerFunc {
return func(f *filterBuilder) {
if parentCount != nil {
f.addJoin("tags_relations", "parents_count", "parents_count.child_id = tags.id")
clause, args := getIntCriterionWhereClause("count(distinct parents_count.parent_id)", *parentCount)
f.addHaving(clause, args...)
}
}
}
func tagChildCountCriterionHandler(qb *tagQueryBuilder, childCount *models.IntCriterionInput) criterionHandlerFunc {
return func(f *filterBuilder) {
if childCount != nil {
f.addJoin("tags_relations", "children_count", "children_count.parent_id = tags.id")
clause, args := getIntCriterionWhereClause("count(distinct children_count.child_id)", *childCount)
f.addHaving(clause, args...)
}
}
}
func (qb *tagQueryBuilder) getDefaultTagSort() string {
return getSort("name", "ASC", "tags")
}

View file

@ -6,6 +6,7 @@ package sqlite_test
import (
"database/sql"
"fmt"
"strconv"
"strings"
"testing"
@ -489,6 +490,198 @@ func verifyTagPerformerCount(t *testing.T, imageCountCriterion models.IntCriteri
})
}
func TestTagQueryParentCount(t *testing.T) {
countCriterion := models.IntCriterionInput{
Value: 1,
Modifier: models.CriterionModifierEquals,
}
verifyTagParentCount(t, countCriterion)
countCriterion.Modifier = models.CriterionModifierNotEquals
verifyTagParentCount(t, countCriterion)
countCriterion.Modifier = models.CriterionModifierLessThan
verifyTagParentCount(t, countCriterion)
countCriterion.Value = 0
countCriterion.Modifier = models.CriterionModifierGreaterThan
verifyTagParentCount(t, countCriterion)
}
func verifyTagParentCount(t *testing.T, sceneCountCriterion models.IntCriterionInput) {
withTxn(func(r models.Repository) error {
qb := r.Tag()
tagFilter := models.TagFilterType{
ParentCount: &sceneCountCriterion,
}
tags := queryTags(t, qb, &tagFilter, nil)
if len(tags) == 0 {
t.Error("Expected at least one tag")
}
for _, tag := range tags {
verifyInt64(t, sql.NullInt64{
Int64: int64(getTagParentCount(tag.ID)),
Valid: true,
}, sceneCountCriterion)
}
return nil
})
}
func TestTagQueryChildCount(t *testing.T) {
countCriterion := models.IntCriterionInput{
Value: 1,
Modifier: models.CriterionModifierEquals,
}
verifyTagChildCount(t, countCriterion)
countCriterion.Modifier = models.CriterionModifierNotEquals
verifyTagChildCount(t, countCriterion)
countCriterion.Modifier = models.CriterionModifierLessThan
verifyTagChildCount(t, countCriterion)
countCriterion.Value = 0
countCriterion.Modifier = models.CriterionModifierGreaterThan
verifyTagChildCount(t, countCriterion)
}
func verifyTagChildCount(t *testing.T, sceneCountCriterion models.IntCriterionInput) {
withTxn(func(r models.Repository) error {
qb := r.Tag()
tagFilter := models.TagFilterType{
ChildCount: &sceneCountCriterion,
}
tags := queryTags(t, qb, &tagFilter, nil)
if len(tags) == 0 {
t.Error("Expected at least one tag")
}
for _, tag := range tags {
verifyInt64(t, sql.NullInt64{
Int64: int64(getTagChildCount(tag.ID)),
Valid: true,
}, sceneCountCriterion)
}
return nil
})
}
func TestTagQueryParent(t *testing.T) {
withTxn(func(r models.Repository) error {
sqb := r.Tag()
tagCriterion := models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdxWithChildTag]),
},
Modifier: models.CriterionModifierIncludes,
}
tagFilter := models.TagFilterType{
Parents: &tagCriterion,
}
tags := queryTags(t, sqb, &tagFilter, nil)
assert.Len(t, tags, 1)
// ensure id is correct
assert.Equal(t, sceneIDs[tagIdxWithParentTag], tags[0].ID)
tagCriterion.Modifier = models.CriterionModifierExcludes
q := getTagStringValue(tagIdxWithParentTag, titleField)
findFilter := models.FindFilterType{
Q: &q,
}
tags = queryTags(t, sqb, &tagFilter, &findFilter)
assert.Len(t, tags, 0)
depth := -1
tagCriterion = models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdxWithGrandChild]),
},
Modifier: models.CriterionModifierIncludes,
Depth: &depth,
}
tags = queryTags(t, sqb, &tagFilter, nil)
assert.Len(t, tags, 2)
depth = 1
tags = queryTags(t, sqb, &tagFilter, nil)
assert.Len(t, tags, 2)
return nil
})
}
func TestTagQueryChild(t *testing.T) {
withTxn(func(r models.Repository) error {
sqb := r.Tag()
tagCriterion := models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdxWithParentTag]),
},
Modifier: models.CriterionModifierIncludes,
}
tagFilter := models.TagFilterType{
Children: &tagCriterion,
}
tags := queryTags(t, sqb, &tagFilter, nil)
assert.Len(t, tags, 1)
// ensure id is correct
assert.Equal(t, sceneIDs[tagIdxWithChildTag], tags[0].ID)
tagCriterion.Modifier = models.CriterionModifierExcludes
q := getTagStringValue(tagIdxWithChildTag, titleField)
findFilter := models.FindFilterType{
Q: &q,
}
tags = queryTags(t, sqb, &tagFilter, &findFilter)
assert.Len(t, tags, 0)
depth := -1
tagCriterion = models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(tagIDs[tagIdxWithGrandParent]),
},
Modifier: models.CriterionModifierIncludes,
Depth: &depth,
}
tags = queryTags(t, sqb, &tagFilter, nil)
assert.Len(t, tags, 2)
depth = 1
tags = queryTags(t, sqb, &tagFilter, nil)
assert.Len(t, tags, 2)
return nil
})
}
func TestTagUpdateTagImage(t *testing.T) {
if err := withTxn(func(r models.Repository) error {
qb := r.Tag()

View file

@ -1,10 +1,10 @@
database: generated.sqlite
scenes: 30000
images: 150000
images: 4000000
galleries: 1500
markers: 300
markers: 3000
performers: 10000
studios: 500
studios: 1500
tags: 1500
naming:
scenes: scene.txt

View file

@ -1,4 +1,4 @@
// uild ignore
// +build ignore
package main
@ -7,6 +7,7 @@ import (
"database/sql"
"fmt"
"log"
"math"
"math/rand"
"os"
"strconv"
@ -20,7 +21,7 @@ import (
"gopkg.in/yaml.v2"
)
const batchSize = 1000
const batchSize = 50000
// create an example database by generating a number of scenes, markers,
// performers, studios and tags, and associating between them all
@ -41,6 +42,8 @@ var txnManager models.TransactionManager
var c *config
func main() {
rand.Seed(time.Now().UnixNano())
var err error
c, err = loadConfig()
if err != nil {
@ -81,6 +84,7 @@ func populateDB() {
makeScenes(c.Scenes)
makeImages(c.Images)
makeGalleries(c.Galleries)
makeMarkers(c.Markers)
}
func withTxn(f func(r models.Repository) error) error {
@ -112,8 +116,25 @@ func makeTags(n int) {
Name: name,
}
_, err := r.Tag().Create(tag)
created, err := r.Tag().Create(tag)
if err != nil {
return err
}
if rand.Intn(100) > 5 {
t, _, err := r.Tag().Query(nil, getRandomFilter(1))
if err != nil {
return err
}
if len(t) > 0 && t[0].ID != created.ID {
if err := r.Tag().UpdateParentTags(created.ID, []int{t[0].ID}); err != nil {
return err
}
}
}
return nil
})
}); err != nil {
panic(err)
@ -184,7 +205,6 @@ func makePerformers(n int) {
func makeScenes(n int) {
logger.Infof("creating %d scenes...", n)
rand.Seed(533)
for i := 0; i < n; {
// do in batches of 1000
batch := i + batchSize
@ -259,7 +279,6 @@ func generateScene(i int) models.Scene {
func makeImages(n int) {
logger.Infof("creating %d images...", n)
rand.Seed(1293)
for i := 0; i < n; {
// do in batches of 1000
batch := i + batchSize
@ -301,7 +320,6 @@ func generateImage(i int) models.Image {
func makeGalleries(n int) {
logger.Infof("creating %d galleries...", n)
rand.Seed(92113)
for i := 0; i < n; {
// do in batches of 1000
batch := i + batchSize
@ -342,8 +360,48 @@ func generateGallery(i int) models.Gallery {
}
}
func makeMarkers(n int) {
logger.Infof("creating %d markers...", n)
for i := 0; i < n; {
// do in batches of 1000
batch := i + batchSize
if err := withTxn(func(r models.Repository) error {
for ; i < batch && i < n; i++ {
marker := generateMarker(i)
marker.SceneID = models.NullInt64(int64(getRandomScene()))
marker.PrimaryTagID = getRandomTags(r, 1, 1)[0]
created, err := r.SceneMarker().Create(marker)
if err != nil {
return err
}
tags := getRandomTags(r, 0, 5)
// remove primary tag
tags = utils.IntExclude(tags, []int{marker.PrimaryTagID})
if err := r.SceneMarker().UpdateTags(created.ID, tags); err != nil {
return err
}
}
logger.Infof("... created %d markers", i)
return nil
}); err != nil {
panic(err)
}
}
}
func generateMarker(i int) models.SceneMarker {
return models.SceneMarker{
Title: names[c.Naming.Scenes].generateName(rand.Intn(7) + 1),
}
}
func getRandomFilter(n int) *models.FindFilterType {
sortBy := "random"
seed := math.Floor(rand.Float64() * math.Pow10(8))
sortBy := fmt.Sprintf("random_%.f", seed)
return &models.FindFilterType{
Sort: &sortBy,
PerPage: &n,
@ -368,7 +426,7 @@ func getRandomStudioID(r models.Repository) sql.NullInt64 {
func makeSceneRelationships(r models.Repository, id int) {
// add tags
tagIDs := getRandomTags(r)
tagIDs := getRandomTags(r, 0, 15)
if len(tagIDs) > 0 {
if err := r.Scene().UpdateTags(id, tagIDs); err != nil {
panic(err)
@ -385,26 +443,33 @@ func makeSceneRelationships(r models.Repository, id int) {
}
func makeImageRelationships(r models.Repository, id int) {
// there are typically many more images. For performance reasons
// only a small proportion should have tags/performers
// add tags
tagIDs := getRandomTags(r)
if rand.Intn(100) == 0 {
tagIDs := getRandomTags(r, 1, 15)
if len(tagIDs) > 0 {
if err := r.Image().UpdateTags(id, tagIDs); err != nil {
panic(err)
}
}
}
// add performers
if rand.Intn(100) <= 1 {
performerIDs := getRandomPerformers(r)
if len(tagIDs) > 0 {
if len(performerIDs) > 0 {
if err := r.Image().UpdatePerformers(id, performerIDs); err != nil {
panic(err)
}
}
}
}
func makeGalleryRelationships(r models.Repository, id int) {
// add tags
tagIDs := getRandomTags(r)
tagIDs := getRandomTags(r, 0, 15)
if len(tagIDs) > 0 {
if err := r.Gallery().UpdateTags(id, tagIDs); err != nil {
panic(err)
@ -450,8 +515,17 @@ func getRandomPerformers(r models.Repository) []int {
return ret
}
func getRandomTags(r models.Repository) []int {
n := rand.Intn(15)
func getRandomScene() int {
return rand.Intn(c.Scenes) + 1
}
func getRandomTags(r models.Repository, min, max int) []int {
var n int
if min == max {
n = min
} else {
n = rand.Intn(max-min) + min
}
var ret []int
// if n > 0 {

View file

@ -1,3 +1,5 @@
// +build ignore
package main
import (

View file

@ -23,6 +23,8 @@ export const HierarchicalLabelValueFilter: React.FC<IHierarchicalLabelValueFilte
criterion.criterionOption.type !== "tags" &&
criterion.criterionOption.type !== "sceneTags" &&
criterion.criterionOption.type !== "performerTags" &&
criterion.criterionOption.type !== "parentTags" &&
criterion.criterionOption.type !== "childTags" &&
criterion.criterionOption.type !== "movies"
)
return null;
@ -53,6 +55,8 @@ export const HierarchicalLabelValueFilter: React.FC<IHierarchicalLabelValueFilte
const optionType =
criterion.criterionOption.type === "studios"
? "include_sub_studios"
: criterion.criterionOption.type === "childTags"
? "include_parent_tags"
: "include_sub_tags";
return {
id: optionType,

View file

@ -20,6 +20,8 @@ export const LabeledIdFilter: React.FC<ILabeledIdFilterProps> = ({
criterion.criterionOption.type !== "tags" &&
criterion.criterionOption.type !== "sceneTags" &&
criterion.criterionOption.type !== "performerTags" &&
criterion.criterionOption.type !== "parentTags" &&
criterion.criterionOption.type !== "childTags" &&
criterion.criterionOption.type !== "movies"
)
return null;

View file

@ -40,6 +40,8 @@ interface ITypeProps {
| "tags"
| "sceneTags"
| "performerTags"
| "parentTags"
| "childTags"
| "movies";
}
interface IFilterProps {

View file

@ -98,6 +98,7 @@
"career_length": "Career Length",
"subsidiary_studios": "Subsidiary Studios",
"sub_tags": "Sub-Tags",
"sub_tag_count": "Sub-Tag Count",
"component_tagger": {
"config": {
"active_instance": "Active stash-box instance:",
@ -545,6 +546,7 @@
"image": "Image",
"image_count": "Image Count",
"images": "Images",
"include_parent_tags": "Include parent tags",
"include_sub_studios": "Include subsidiary studios",
"include_sub_tags": "Include sub-tags",
"instagram": "Instagram",
@ -589,6 +591,7 @@
},
"parent_studios": "Parent Studios",
"parent_tags": "Parent Tags",
"parent_tag_count": "Parent Tag Count",
"path": "Path",
"performer": "Performer",
"performer_count": "Performer Count",

View file

@ -27,6 +27,8 @@ import { PerformersCriterion } from "./performers";
import { AverageResolutionCriterion, ResolutionCriterion } from "./resolution";
import { StudiosCriterion, ParentStudiosCriterion } from "./studios";
import {
ChildTagsCriterionOption,
ParentTagsCriterionOption,
PerformerTagsCriterionOption,
SceneTagsCriterionOption,
TagsCriterion,
@ -98,6 +100,10 @@ export function makeCriteria(type: CriterionType = "none") {
return new TagsCriterion(SceneTagsCriterionOption);
case "performerTags":
return new TagsCriterion(PerformerTagsCriterionOption);
case "parentTags":
return new TagsCriterion(ParentTagsCriterionOption);
case "childTags":
return new TagsCriterion(ChildTagsCriterionOption);
case "performers":
return new PerformersCriterion();
case "studios":
@ -145,5 +151,21 @@ export function makeCriteria(type: CriterionType = "none") {
return new StringCriterion(new StringCriterionOption(type, type));
case "interactive":
return new InteractiveCriterion();
case "parent_tag_count":
return new NumberCriterion(
new MandatoryNumberCriterionOption(
"parent_tag_count",
"parent_tag_count",
"parent_count"
)
);
case "child_tag_count":
return new NumberCriterion(
new MandatoryNumberCriterionOption(
"sub_tag_count",
"child_tag_count",
"child_count"
)
);
}
}

View file

@ -23,3 +23,15 @@ export const PerformerTagsCriterionOption = new ILabeledIdCriterionOption(
"performer_tags",
true
);
export const ParentTagsCriterionOption = new ILabeledIdCriterionOption(
"parent_tags",
"parentTags",
"parents",
true
);
export const ChildTagsCriterionOption = new ILabeledIdCriterionOption(
"sub_tags",
"childTags",
"children",
true
);

View file

@ -2,10 +2,15 @@ import {
createMandatoryNumberCriterionOption,
createMandatoryStringCriterionOption,
createStringCriterionOption,
MandatoryNumberCriterionOption,
} from "./criteria/criterion";
import { TagIsMissingCriterionOption } from "./criteria/is-missing";
import { ListFilterOptions } from "./filter-options";
import { DisplayMode } from "./types";
import {
ChildTagsCriterionOption,
ParentTagsCriterionOption,
} from "./criteria/tags";
const defaultSortBy = "name";
const sortByOptions = ["name", "random"]
@ -43,6 +48,18 @@ const criterionOptions = [
createMandatoryNumberCriterionOption("gallery_count"),
createMandatoryNumberCriterionOption("performer_count"),
createMandatoryNumberCriterionOption("marker_count"),
ParentTagsCriterionOption,
new MandatoryNumberCriterionOption(
"parent_tag_count",
"parent_tag_count",
"parent_count"
),
ChildTagsCriterionOption,
new MandatoryNumberCriterionOption(
"sub_tag_count",
"child_tag_count",
"child_count"
),
];
export const TagListFilterOptions = new ListFilterOptions(

View file

@ -75,6 +75,8 @@ export type CriterionType =
| "tags"
| "sceneTags"
| "performerTags"
| "parentTags"
| "childTags"
| "tag_count"
| "performers"
| "studios"
@ -114,4 +116,6 @@ export type CriterionType =
| "galleryChecksum"
| "phash"
| "director"
| "synopsis";
| "synopsis"
| "parent_tag_count"
| "child_tag_count";