Backup_Repos/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-06 08:26:00 +01:00
Code Issues Projects Releases Wiki Activity

Performance improvements (#2925)

Browse source 
* Add sqlite_stat4 build tag
* Simplify studio filter criterion queries
* Prevent useList loading data before filter initialized
This commit is contained in:
DingDongSoLong4 2022-09-30 02:49:51 +02:00 committed by GitHub
parent d274f86390
commit 25bc750295
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 183 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -54,7 +54,7 @@ build: pre-build
build:
$(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/internal/api.version=$(STASH_VERSION)' -X 'github.com/stashapp/stash/internal/api.buildstamp=$(BUILD_DATE)' -X 'github.com/stashapp/stash/internal/api.githash=$(GITHASH)')
$(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/internal/manager/config.officialBuild=$(OFFICIAL_BUILD)')
go build $(OUTPUT) -mod=vendor -v -tags "sqlite_omit_load_extension osusergo netgo" $(GO_BUILD_FLAGS) -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS) $(PLATFORM_SPECIFIC_LDFLAGS)" ./cmd/stash
go build $(OUTPUT) -mod=vendor -v -tags "sqlite_omit_load_extension sqlite_stat4 osusergo netgo" $(GO_BUILD_FLAGS) -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS) $(PLATFORM_SPECIFIC_LDFLAGS)" ./cmd/stash
# strips debug symbols from the release build
build-release: EXTRA_LDFLAGS := -s -w

13
pkg/sqlite/filter.go
View file

@ -744,7 +744,6 @@ type hierarchicalMultiCriterionHandlerBuilder struct {
foreignTable string
foreignFK string
derivedTable string
parentFK string
relationsTable string
}
@ -867,9 +866,15 @@ func (m *hierarchicalMultiCriterionHandlerBuilder) handler(criterion *models.Hie
valuesClause := getHierarchicalValues(ctx, m.tx, criterion.Value, m.foreignTable, m.relationsTable, m.parentFK, criterion.Depth)
f.addLeftJoin("(SELECT column1 AS root_id, column2 AS item_id FROM ("+valuesClause+"))", m.derivedTable, fmt.Sprintf("%s.item_id = %s.%s", m.derivedTable, m.primaryTable, m.foreignFK))
addHierarchicalConditionClauses(f, criterion, m.derivedTable, "root_id")
switch criterion.Modifier {
case models.CriterionModifierIncludes:
f.addWhere(fmt.Sprintf("%s.%s IN (SELECT column2 FROM (%s))", m.primaryTable, m.foreignFK, valuesClause))
case models.CriterionModifierIncludesAll:
f.addWhere(fmt.Sprintf("%s.%s IN (SELECT column2 FROM (%s))", m.primaryTable, m.foreignFK, valuesClause))
f.addHaving(fmt.Sprintf("count(distinct %s.%s) IS %d", m.primaryTable, m.foreignFK, len(criterion.Value)))
case models.CriterionModifierExcludes:
f.addWhere(fmt.Sprintf("%s.%s NOT IN (SELECT column2 FROM (%s)) OR %[1]s.%[2]s IS NULL", m.primaryTable, m.foreignFK, valuesClause))
}
}
}
}

1
pkg/sqlite/gallery.go
View file

@ -938,7 +938,6 @@ func galleryStudioCriterionHandler(qb *GalleryStore, studios *models.Hierarchica
primaryTable: galleryTable,
foreignTable: studioTable,
foreignFK: studioIDColumn,
derivedTable: "studio",
parentFK: "parent_id",
}

1
pkg/sqlite/image.go
View file

@ -922,7 +922,6 @@ func imageStudioCriterionHandler(qb *ImageStore, studios *models.HierarchicalMul
primaryTable: imageTable,
foreignTable: studioTable,
foreignFK: studioIDColumn,
derivedTable: "studio",
parentFK: "parent_id",
}

1
pkg/sqlite/movies.go
View file

@ -218,7 +218,6 @@ func movieStudioCriterionHandler(qb *movieQueryBuilder, studios *models.Hierarch
primaryTable: movieTable,
foreignTable: studioTable,
foreignFK: studioIDColumn,
derivedTable: "studio",
parentFK: "parent_id",
}

1
pkg/sqlite/scene.go
View file

@ -1244,7 +1244,6 @@ func sceneStudioCriterionHandler(qb *SceneStore, studios *models.HierarchicalMul
primaryTable: sceneTable,
foreignTable: studioTable,
foreignFK: studioIDColumn,
derivedTable: "studio",
parentFK: "parent_id",
}

100
pkg/sqlite/scene_test.go
View file

@ -1787,9 +1787,9 @@ func TestSceneCountByPerformerID(t *testing.T) {
}
func scenesToIDs(i []*models.Scene) []int {
var ret []int
for _, ii := range i {
ret = append(ret, ii.ID)
ret := make([]int, len(i))
for i, v := range i {
ret[i] = v.ID
}
return ret
@ -3304,43 +3304,73 @@ func TestSceneQueryPerformerTags(t *testing.T) {
}
func TestSceneQueryStudio(t *testing.T) {
withTxn(func(ctx context.Context) error {
sqb := db.Scene
studioCriterion := models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(studioIDs[studioIdxWithScene]),
tests := []struct {
name string
q string
studioCriterion models.HierarchicalMultiCriterionInput
expectedIDs []int
wantErr bool
}{
{
"includes",
"",
models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(studioIDs[studioIdxWithScene]),
},
Modifier: models.CriterionModifierIncludes,
},
Modifier: models.CriterionModifierIncludes,
}
sceneFilter := models.SceneFilterType{
Studios: &studioCriterion,
}
scenes := queryScene(ctx, t, sqb, &sceneFilter, nil)
assert.Len(t, scenes, 1)
// ensure id is correct
assert.Equal(t, sceneIDs[sceneIdxWithStudio], scenes[0].ID)
studioCriterion = models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(studioIDs[studioIdxWithScene]),
[]int{sceneIDs[sceneIdxWithStudio]},
false,
},
{
"excludes",
getSceneStringValue(sceneIdxWithStudio, titleField),
models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(studioIDs[studioIdxWithScene]),
},
Modifier: models.CriterionModifierExcludes,
},
Modifier: models.CriterionModifierExcludes,
}
[]int{},
false,
},
{
"excludes includes null",
getSceneStringValue(sceneIdxWithGallery, titleField),
models.HierarchicalMultiCriterionInput{
Value: []string{
strconv.Itoa(studioIDs[studioIdxWithScene]),
},
Modifier: models.CriterionModifierExcludes,
},
[]int{sceneIDs[sceneIdxWithGallery]},
false,
},
}
q := getSceneStringValue(sceneIdxWithStudio, titleField)
findFilter := models.FindFilterType{
Q: &q,
}
qb := db.Scene
scenes = queryScene(ctx, t, sqb, &sceneFilter, &findFilter)
assert.Len(t, scenes, 0)
for _, tt := range tests {
runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
studioCriterion := tt.studioCriterion
return nil
})
sceneFilter := models.SceneFilterType{
Studios: &studioCriterion,
}
var findFilter *models.FindFilterType
if tt.q != "" {
findFilter = &models.FindFilterType{
Q: &tt.q,
}
}
scenes := queryScene(ctx, t, qb, &sceneFilter, findFilter)
assert.ElementsMatch(t, scenesToIDs(scenes), tt.expectedIDs)
})
}
}
func TestSceneQueryStudioDepth(t *testing.T) {

56
ui/v2.5/src/core/StashService.ts
View file

@ -65,11 +65,12 @@ export const useFindDefaultFilter = (mode: GQL.FilterMode) =>
},
});
export const useFindGalleries = (filter: ListFilterModel) =>
export const useFindGalleries = (filter?: ListFilterModel) =>
GQL.useFindGalleriesQuery({
skip: filter === undefined,
variables: {
filter: filter.makeFindFilter(),
gallery_filter: filter.makeFilter(),
filter: filter?.makeFindFilter(),
gallery_filter: filter?.makeFilter(),
},
});
@ -82,11 +83,12 @@ export const queryFindGalleries = (filter: ListFilterModel) =>
},
});
export const useFindScenes = (filter: ListFilterModel) =>
export const useFindScenes = (filter?: ListFilterModel) =>
GQL.useFindScenesQuery({
skip: filter === undefined,
variables: {
filter: filter.makeFindFilter(),
scene_filter: filter.makeFilter(),
filter: filter?.makeFindFilter(),
scene_filter: filter?.makeFilter(),
},
});
@ -107,11 +109,12 @@ export const queryFindScenesByID = (sceneIDs: number[]) =>
},
});
export const useFindSceneMarkers = (filter: ListFilterModel) =>
export const useFindSceneMarkers = (filter?: ListFilterModel) =>
GQL.useFindSceneMarkersQuery({
skip: filter === undefined,
variables: {
filter: filter.makeFindFilter(),
scene_marker_filter: filter.makeFilter(),
filter: filter?.makeFindFilter(),
scene_marker_filter: filter?.makeFilter(),
},
});
@ -124,11 +127,12 @@ export const queryFindSceneMarkers = (filter: ListFilterModel) =>
},
});
export const useFindImages = (filter: ListFilterModel) =>
export const useFindImages = (filter?: ListFilterModel) =>
GQL.useFindImagesQuery({
skip: filter === undefined,
variables: {
filter: filter.makeFindFilter(),
image_filter: filter.makeFilter(),
filter: filter?.makeFindFilter(),
image_filter: filter?.makeFilter(),
},
});
@ -141,11 +145,12 @@ export const queryFindImages = (filter: ListFilterModel) =>
},
});
export const useFindStudios = (filter: ListFilterModel) =>
export const useFindStudios = (filter?: ListFilterModel) =>
GQL.useFindStudiosQuery({
skip: filter === undefined,
variables: {
filter: filter.makeFindFilter(),
studio_filter: filter.makeFilter(),
filter: filter?.makeFindFilter(),
studio_filter: filter?.makeFilter(),
},
});
@ -158,11 +163,12 @@ export const queryFindStudios = (filter: ListFilterModel) =>
},
});
export const useFindMovies = (filter: ListFilterModel) =>
export const useFindMovies = (filter?: ListFilterModel) =>
GQL.useFindMoviesQuery({
skip: filter === undefined,
variables: {
filter: filter.makeFindFilter(),
movie_filter: filter.makeFilter(),
filter: filter?.makeFindFilter(),
movie_filter: filter?.makeFilter(),
},
});
@ -175,19 +181,21 @@ export const queryFindMovies = (filter: ListFilterModel) =>
},
});
export const useFindPerformers = (filter: ListFilterModel) =>
export const useFindPerformers = (filter?: ListFilterModel) =>
GQL.useFindPerformersQuery({
skip: filter === undefined,
variables: {
filter: filter.makeFindFilter(),
performer_filter: filter.makeFilter(),
filter: filter?.makeFindFilter(),
performer_filter: filter?.makeFilter(),
},
});
export const useFindTags = (filter: ListFilterModel) =>
export const useFindTags = (filter?: ListFilterModel) =>
GQL.useFindTagsQuery({
skip: filter === undefined,
variables: {
filter: filter.makeFindFilter(),
tag_filter: filter.makeFilter(),
filter: filter?.makeFindFilter(),
tag_filter: filter?.makeFilter(),
},
});

125
ui/v2.5/src/hooks/ListHook.tsx
View file

@ -94,7 +94,7 @@ export interface IListHookOperation<T> {
result: T,
filter: ListFilterModel,
selectedIds: Set<string>
) => void;
) => Promise<void>;
isDisplayed?: (
result: T,
filter: ListFilterModel,
@ -160,20 +160,20 @@ interface IQueryResult {
interface IQuery<T extends IQueryResult, T2 extends IDataItem> {
filterMode: FilterMode;
useData: (filter: ListFilterModel) => T;
useData: (filter?: ListFilterModel) => T;
getData: (data: T) => T2[];
getCount: (data: T) => number;
getMetadataByline: (data: T) => React.ReactNode;
}
interface IRenderListProps {
filter: ListFilterModel;
filter?: ListFilterModel;
filterOptions: ListFilterOptions;
onChangePage: (page: number) => void;
updateFilter: (filter: ListFilterModel) => void;
}
const RenderList = <
const useRenderList = <
QueryResult extends IQueryResult,
QueryData extends IDataItem
>({
@ -200,31 +200,44 @@ const RenderList = <
const [isEditDialogOpen, setIsEditDialogOpen] = useState(false);
const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false);
const [selectedIds, setSelectedIds] = useState<Set<string>>(new Set());
const [lastClickedId, setLastClickedId] = useState<string | undefined>();
const [lastClickedId, setLastClickedId] = useState<string>();
const [editingCriterion, setEditingCriterion] = useState<
Criterion<CriterionValue> | undefined
>(undefined);
Criterion<CriterionValue>
>();
const [newCriterion, setNewCriterion] = useState(false);
const result = useData(filter);
const totalCount = getCount(result);
const metadataByline = getMetadataByline(result);
const items = getData(result);
const pages = Math.ceil(totalCount / filter.itemsPerPage);
// handle case where page is more than there are pages
useEffect(() => {
if (filter === undefined) return;
const pages = Math.ceil(totalCount / filter.itemsPerPage);
if (pages > 0 && filter.currentPage > pages) {
onChangePage(pages);
}
}, [pages, filter.currentPage, onChangePage]);
}, [filter, onChangePage, totalCount]);
// set up hotkeys
useEffect(() => {
if (filter === undefined) return;
Mousetrap.bind("f", () => setNewCriterion(true));
return () => {
Mousetrap.unbind("f");
};
}, [filter]);
useEffect(() => {
if (filter === undefined) return;
const pages = Math.ceil(totalCount / filter.itemsPerPage);
Mousetrap.bind("right", () => {
const maxPage = totalCount / filter.itemsPerPage;
if (filter.currentPage < maxPage) {
if (filter.currentPage < pages) {
onChangePage(filter.currentPage + 1);
}
});
@ -234,25 +247,18 @@ const RenderList = <
}
});
Mousetrap.bind("shift+right", () => {
const maxPage = totalCount / filter.itemsPerPage + 1;
onChangePage(Math.min(maxPage, filter.currentPage + 10));
onChangePage(Math.min(pages, filter.currentPage + 10));
});
Mousetrap.bind("shift+left", () => {
onChangePage(Math.max(1, filter.currentPage - 10));
});
Mousetrap.bind("ctrl+end", () => {
const maxPage = totalCount / filter.itemsPerPage + 1;
onChangePage(maxPage);
onChangePage(pages);
});
Mousetrap.bind("ctrl+home", () => {
onChangePage(1);
});
let unbindExtras: () => void;
if (addKeybinds) {
unbindExtras = addKeybinds(result, filter, selectedIds);
}
return () => {
Mousetrap.unbind("right");
Mousetrap.unbind("left");
@ -260,12 +266,22 @@ const RenderList = <
Mousetrap.unbind("shift+left");
Mousetrap.unbind("ctrl+end");
Mousetrap.unbind("ctrl+home");
if (unbindExtras) {
unbindExtras();
}
};
});
}, [filter, onChangePage, totalCount]);
useEffect(() => {
if (filter === undefined) return;
if (addKeybinds) {
const unbindExtras = addKeybinds(result, filter, selectedIds);
return () => {
unbindExtras();
};
}
}, [addKeybinds, filter, result, selectedIds]);
// Don't continue if filter is undefined
// There are no hooks below this point so this is valid
if (filter === undefined) return;
function singleSelect(id: string, selected: boolean) {
setLastClickedId(id);
@ -334,24 +350,24 @@ const RenderList = <
setLastClickedId(undefined);
}
function onSelectNone() {
const onSelectNone = () => {
const newSelectedIds: Set<string> = new Set();
setSelectedIds(newSelectedIds);
setLastClickedId(undefined);
}
};
function onChangeZoom(newZoomIndex: number) {
const onChangeZoom = (newZoomIndex: number) => {
const newFilter = cloneDeep(filter);
newFilter.zoomIndex = newZoomIndex;
updateFilter(newFilter);
}
};
function onOperationClicked(o: IListHookOperation<QueryResult>) {
o.onClick(result, filter, selectedIds);
const onOperationClicked = async (o: IListHookOperation<QueryResult>) => {
await o.onClick(result, filter, selectedIds);
if (o.postRefetch) {
result.refetch();
}
}
};
const operations =
otherOperations &&
@ -409,11 +425,12 @@ const RenderList = <
/>
);
function maybeRenderContent() {
const maybeRenderContent = () => {
if (result.loading || result.error) {
return;
}
const pages = Math.ceil(totalCount / filter.itemsPerPage);
return (
<>
{renderPagination()}
@ -433,18 +450,18 @@ const RenderList = <
{renderPagination()}
</>
);
}
};
function onChangeDisplayMode(displayMode: DisplayMode) {
const onChangeDisplayMode = (displayMode: DisplayMode) => {
const newFilter = cloneDeep(filter);
newFilter.displayMode = displayMode;
updateFilter(newFilter);
}
};
function onAddCriterion(
const onAddCriterion = (
criterion: Criterion<CriterionValue>,
oldId?: string
) {
) => {
const newFilter = cloneDeep(filter);
// Find if we are editing an existing criteria, then modify that. Or create a new one.
@ -468,22 +485,22 @@ const RenderList = <
updateFilter(newFilter);
setEditingCriterion(undefined);
setNewCriterion(false);
}
};
function onRemoveCriterion(removedCriterion: Criterion<CriterionValue>) {
const onRemoveCriterion = (removedCriterion: Criterion<CriterionValue>) => {
const newFilter = cloneDeep(filter);
newFilter.criteria = newFilter.criteria.filter(
(criterion) => criterion.getId() !== removedCriterion.getId()
);
newFilter.currentPage = 1;
updateFilter(newFilter);
}
};
function updateCriteria(c: Criterion<CriterionValue>[]) {
const updateCriteria = (c: Criterion<CriterionValue>[]) => {
const newFilter = cloneDeep(filter);
newFilter.criteria = c.slice();
setNewCriterion(false);
}
};
function onCancelAddCriterion() {
setEditingCriterion(undefined);
@ -732,10 +749,14 @@ const useList = <QueryResult extends IQueryResult, QueryData extends IDataItem>(
);
const renderFilter = useMemo(() => {
return !options.filterHook ? filter : options.filterHook(cloneDeep(filter));
}, [filter, options]);
if (filterInitialised) {
return options.filterHook
? options.filterHook(cloneDeep(filter))
: filter;
}
}, [filterInitialised, filter, options]);
const { contentTemplate, onSelectChange } = RenderList({
const renderList = useRenderList({
...options,
filter: renderFilter,
filterOptions,
@ -743,12 +764,18 @@ const useList = <QueryResult extends IQueryResult, QueryData extends IDataItem>(
updateFilter,
});
const template = !filterInitialised ? (
<LoadingIndicator />
const template = renderList ? (
renderList.contentTemplate
) : (
<>{contentTemplate}</>
<LoadingIndicator />
);
function onSelectChange(id: string, selected: boolean, shiftKey: boolean) {
if (renderList) {
renderList.onSelectChange(id, selected, shiftKey);
}
}
return {
filter,
template,