Performance improvements (#2925)

* Add sqlite_stat4 build tag
* Simplify studio filter criterion queries
* Prevent useList loading data before filter initialized

Makefile

@@ -54,7 +54,7 @@ build: pre-build
 build:
 	$(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/internal/api.version=$(STASH_VERSION)' -X 'github.com/stashapp/stash/internal/api.buildstamp=$(BUILD_DATE)' -X 'github.com/stashapp/stash/internal/api.githash=$(GITHASH)')
 	$(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/internal/manager/config.officialBuild=$(OFFICIAL_BUILD)')
-	go build $(OUTPUT) -mod=vendor -v -tags "sqlite_omit_load_extension osusergo netgo" $(GO_BUILD_FLAGS) -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS) $(PLATFORM_SPECIFIC_LDFLAGS)" ./cmd/stash
+	go build $(OUTPUT) -mod=vendor -v -tags "sqlite_omit_load_extension sqlite_stat4 osusergo netgo" $(GO_BUILD_FLAGS) -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS) $(PLATFORM_SPECIFIC_LDFLAGS)" ./cmd/stash
 
 # strips debug symbols from the release build
 build-release: EXTRA_LDFLAGS := -s -w

pkg/sqlite/filter.go

@@ -744,7 +744,6 @@ type hierarchicalMultiCriterionHandlerBuilder struct {
 	foreignTable string
 	foreignFK    string
 
-	derivedTable   string
 	parentFK       string
 	relationsTable string
 }
@@ -867,9 +866,15 @@ func (m *hierarchicalMultiCriterionHandlerBuilder) handler(criterion *models.Hie
 
 			valuesClause := getHierarchicalValues(ctx, m.tx, criterion.Value, m.foreignTable, m.relationsTable, m.parentFK, criterion.Depth)
 
-			f.addLeftJoin("(SELECT column1 AS root_id, column2 AS item_id FROM ("+valuesClause+"))", m.derivedTable, fmt.Sprintf("%s.item_id = %s.%s", m.derivedTable, m.primaryTable, m.foreignFK))
+			switch criterion.Modifier {
-
+			case models.CriterionModifierIncludes:
-			addHierarchicalConditionClauses(f, criterion, m.derivedTable, "root_id")
+				f.addWhere(fmt.Sprintf("%s.%s IN (SELECT column2 FROM (%s))", m.primaryTable, m.foreignFK, valuesClause))
+			case models.CriterionModifierIncludesAll:
+				f.addWhere(fmt.Sprintf("%s.%s IN (SELECT column2 FROM (%s))", m.primaryTable, m.foreignFK, valuesClause))
+				f.addHaving(fmt.Sprintf("count(distinct %s.%s) IS %d", m.primaryTable, m.foreignFK, len(criterion.Value)))
+			case models.CriterionModifierExcludes:
+				f.addWhere(fmt.Sprintf("%s.%s NOT IN (SELECT column2 FROM (%s)) OR %[1]s.%[2]s IS NULL", m.primaryTable, m.foreignFK, valuesClause))
+			}
 		}
 	}
 }

pkg/sqlite/gallery.go

@@ -938,7 +938,6 @@ func galleryStudioCriterionHandler(qb *GalleryStore, studios *models.Hierarchica
 		primaryTable: galleryTable,
 		foreignTable: studioTable,
 		foreignFK:    studioIDColumn,
-		derivedTable: "studio",
 		parentFK:     "parent_id",
 	}
 

pkg/sqlite/image.go

@@ -922,7 +922,6 @@ func imageStudioCriterionHandler(qb *ImageStore, studios *models.HierarchicalMul
 		primaryTable: imageTable,
 		foreignTable: studioTable,
 		foreignFK:    studioIDColumn,
-		derivedTable: "studio",
 		parentFK:     "parent_id",
 	}
 

pkg/sqlite/movies.go

@@ -218,7 +218,6 @@ func movieStudioCriterionHandler(qb *movieQueryBuilder, studios *models.Hierarch
 		primaryTable: movieTable,
 		foreignTable: studioTable,
 		foreignFK:    studioIDColumn,
-		derivedTable: "studio",
 		parentFK:     "parent_id",
 	}
 

pkg/sqlite/scene.go

@@ -1244,7 +1244,6 @@ func sceneStudioCriterionHandler(qb *SceneStore, studios *models.HierarchicalMul
 		primaryTable: sceneTable,
 		foreignTable: studioTable,
 		foreignFK:    studioIDColumn,
-		derivedTable: "studio",
 		parentFK:     "parent_id",
 	}
 

pkg/sqlite/scene_test.go

@@ -1787,9 +1787,9 @@ func TestSceneCountByPerformerID(t *testing.T) {
 }
 
 func scenesToIDs(i []*models.Scene) []int {
-	var ret []int
+	ret := make([]int, len(i))
-	for _, ii := range i {
+	for i, v := range i {
-		ret = append(ret, ii.ID)
+		ret[i] = v.ID
 	}
 
 	return ret
@@ -3304,44 +3304,74 @@ func TestSceneQueryPerformerTags(t *testing.T) {
 }
 
 func TestSceneQueryStudio(t *testing.T) {
-	withTxn(func(ctx context.Context) error {
+	tests := []struct {
-		sqb := db.Scene
+		name            string
-		studioCriterion := models.HierarchicalMultiCriterionInput{
+		q               string
+		studioCriterion models.HierarchicalMultiCriterionInput
+		expectedIDs     []int
+		wantErr         bool
+	}{
+		{
+			"includes",
+			"",
+			models.HierarchicalMultiCriterionInput{
 				Value: []string{
 					strconv.Itoa(studioIDs[studioIdxWithScene]),
 				},
 				Modifier: models.CriterionModifierIncludes,
+			},
+			[]int{sceneIDs[sceneIdxWithStudio]},
+			false,
+		},
+		{
+			"excludes",
+			getSceneStringValue(sceneIdxWithStudio, titleField),
+			models.HierarchicalMultiCriterionInput{
+				Value: []string{
+					strconv.Itoa(studioIDs[studioIdxWithScene]),
+				},
+				Modifier: models.CriterionModifierExcludes,
+			},
+			[]int{},
+			false,
+		},
+		{
+			"excludes includes null",
+			getSceneStringValue(sceneIdxWithGallery, titleField),
+			models.HierarchicalMultiCriterionInput{
+				Value: []string{
+					strconv.Itoa(studioIDs[studioIdxWithScene]),
+				},
+				Modifier: models.CriterionModifierExcludes,
+			},
+			[]int{sceneIDs[sceneIdxWithGallery]},
+			false,
+		},
 	}
 
+	qb := db.Scene
+
+	for _, tt := range tests {
+		runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) {
+			studioCriterion := tt.studioCriterion
+
 			sceneFilter := models.SceneFilterType{
 				Studios: &studioCriterion,
 			}
 
-		scenes := queryScene(ctx, t, sqb, &sceneFilter, nil)
+			var findFilter *models.FindFilterType
-
+			if tt.q != "" {
-		assert.Len(t, scenes, 1)
+				findFilter = &models.FindFilterType{
-
+					Q: &tt.q,
-		// ensure id is correct
+				}
-		assert.Equal(t, sceneIDs[sceneIdxWithStudio], scenes[0].ID)
-
-		studioCriterion = models.HierarchicalMultiCriterionInput{
-			Value: []string{
-				strconv.Itoa(studioIDs[studioIdxWithScene]),
-			},
-			Modifier: models.CriterionModifierExcludes,
 			}
 
-		q := getSceneStringValue(sceneIdxWithStudio, titleField)
+			scenes := queryScene(ctx, t, qb, &sceneFilter, findFilter)
-		findFilter := models.FindFilterType{
-			Q: &q,
-		}
 
-		scenes = queryScene(ctx, t, sqb, &sceneFilter, &findFilter)
+			assert.ElementsMatch(t, scenesToIDs(scenes), tt.expectedIDs)
-		assert.Len(t, scenes, 0)
-
-		return nil
 		})
 	}
+}
 
 func TestSceneQueryStudioDepth(t *testing.T) {
 	withTxn(func(ctx context.Context) error {

ui/v2.5/src/core/StashService.ts

@@ -65,11 +65,12 @@ export const useFindDefaultFilter = (mode: GQL.FilterMode) =>
     },
   });
 
-export const useFindGalleries = (filter: ListFilterModel) =>
+export const useFindGalleries = (filter?: ListFilterModel) =>
   GQL.useFindGalleriesQuery({
+    skip: filter === undefined,
     variables: {
-      filter: filter.makeFindFilter(),
+      filter: filter?.makeFindFilter(),
-      gallery_filter: filter.makeFilter(),
+      gallery_filter: filter?.makeFilter(),
     },
   });
 
@@ -82,11 +83,12 @@ export const queryFindGalleries = (filter: ListFilterModel) =>
     },
   });
 
-export const useFindScenes = (filter: ListFilterModel) =>
+export const useFindScenes = (filter?: ListFilterModel) =>
   GQL.useFindScenesQuery({
+    skip: filter === undefined,
     variables: {
-      filter: filter.makeFindFilter(),
+      filter: filter?.makeFindFilter(),
-      scene_filter: filter.makeFilter(),
+      scene_filter: filter?.makeFilter(),
     },
   });
 
@@ -107,11 +109,12 @@ export const queryFindScenesByID = (sceneIDs: number[]) =>
     },
   });
 
-export const useFindSceneMarkers = (filter: ListFilterModel) =>
+export const useFindSceneMarkers = (filter?: ListFilterModel) =>
   GQL.useFindSceneMarkersQuery({
+    skip: filter === undefined,
     variables: {
-      filter: filter.makeFindFilter(),
+      filter: filter?.makeFindFilter(),
-      scene_marker_filter: filter.makeFilter(),
+      scene_marker_filter: filter?.makeFilter(),
     },
   });
 
@@ -124,11 +127,12 @@ export const queryFindSceneMarkers = (filter: ListFilterModel) =>
     },
   });
 
-export const useFindImages = (filter: ListFilterModel) =>
+export const useFindImages = (filter?: ListFilterModel) =>
   GQL.useFindImagesQuery({
+    skip: filter === undefined,
     variables: {
-      filter: filter.makeFindFilter(),
+      filter: filter?.makeFindFilter(),
-      image_filter: filter.makeFilter(),
+      image_filter: filter?.makeFilter(),
     },
   });
 
@@ -141,11 +145,12 @@ export const queryFindImages = (filter: ListFilterModel) =>
     },
   });
 
-export const useFindStudios = (filter: ListFilterModel) =>
+export const useFindStudios = (filter?: ListFilterModel) =>
   GQL.useFindStudiosQuery({
+    skip: filter === undefined,
     variables: {
-      filter: filter.makeFindFilter(),
+      filter: filter?.makeFindFilter(),
-      studio_filter: filter.makeFilter(),
+      studio_filter: filter?.makeFilter(),
     },
   });
 
@@ -158,11 +163,12 @@ export const queryFindStudios = (filter: ListFilterModel) =>
     },
   });
 
-export const useFindMovies = (filter: ListFilterModel) =>
+export const useFindMovies = (filter?: ListFilterModel) =>
   GQL.useFindMoviesQuery({
+    skip: filter === undefined,
     variables: {
-      filter: filter.makeFindFilter(),
+      filter: filter?.makeFindFilter(),
-      movie_filter: filter.makeFilter(),
+      movie_filter: filter?.makeFilter(),
     },
   });
 
@@ -175,19 +181,21 @@ export const queryFindMovies = (filter: ListFilterModel) =>
     },
   });
 
-export const useFindPerformers = (filter: ListFilterModel) =>
+export const useFindPerformers = (filter?: ListFilterModel) =>
   GQL.useFindPerformersQuery({
+    skip: filter === undefined,
     variables: {
-      filter: filter.makeFindFilter(),
+      filter: filter?.makeFindFilter(),
-      performer_filter: filter.makeFilter(),
+      performer_filter: filter?.makeFilter(),
     },
   });
 
-export const useFindTags = (filter: ListFilterModel) =>
+export const useFindTags = (filter?: ListFilterModel) =>
   GQL.useFindTagsQuery({
+    skip: filter === undefined,
     variables: {
-      filter: filter.makeFindFilter(),
+      filter: filter?.makeFindFilter(),
-      tag_filter: filter.makeFilter(),
+      tag_filter: filter?.makeFilter(),
     },
   });
 

ui/v2.5/src/hooks/ListHook.tsx

@@ -94,7 +94,7 @@ export interface IListHookOperation<T> {
     result: T,
     filter: ListFilterModel,
     selectedIds: Set<string>
-  ) => void;
+  ) => Promise<void>;
   isDisplayed?: (
     result: T,
     filter: ListFilterModel,
@@ -160,20 +160,20 @@ interface IQueryResult {
 
 interface IQuery<T extends IQueryResult, T2 extends IDataItem> {
   filterMode: FilterMode;
-  useData: (filter: ListFilterModel) => T;
+  useData: (filter?: ListFilterModel) => T;
   getData: (data: T) => T2[];
   getCount: (data: T) => number;
   getMetadataByline: (data: T) => React.ReactNode;
 }
 
 interface IRenderListProps {
-  filter: ListFilterModel;
+  filter?: ListFilterModel;
   filterOptions: ListFilterOptions;
   onChangePage: (page: number) => void;
   updateFilter: (filter: ListFilterModel) => void;
 }
 
-const RenderList = <
+const useRenderList = <
   QueryResult extends IQueryResult,
   QueryData extends IDataItem
 >({
@@ -200,31 +200,44 @@ const RenderList = <
   const [isEditDialogOpen, setIsEditDialogOpen] = useState(false);
   const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false);
   const [selectedIds, setSelectedIds] = useState<Set<string>>(new Set());
-  const [lastClickedId, setLastClickedId] = useState<string | undefined>();
+  const [lastClickedId, setLastClickedId] = useState<string>();
 
   const [editingCriterion, setEditingCriterion] = useState<
-    Criterion<CriterionValue> | undefined
+    Criterion<CriterionValue>
-  >(undefined);
+  >();
   const [newCriterion, setNewCriterion] = useState(false);
 
   const result = useData(filter);
   const totalCount = getCount(result);
   const metadataByline = getMetadataByline(result);
   const items = getData(result);
-  const pages = Math.ceil(totalCount / filter.itemsPerPage);
 
   // handle case where page is more than there are pages
   useEffect(() => {
+    if (filter === undefined) return;
+
+    const pages = Math.ceil(totalCount / filter.itemsPerPage);
     if (pages > 0 && filter.currentPage > pages) {
       onChangePage(pages);
     }
-  }, [pages, filter.currentPage, onChangePage]);
+  }, [filter, onChangePage, totalCount]);
 
+  // set up hotkeys
   useEffect(() => {
+    if (filter === undefined) return;
+
     Mousetrap.bind("f", () => setNewCriterion(true));
+
+    return () => {
+      Mousetrap.unbind("f");
+    };
+  }, [filter]);
+  useEffect(() => {
+    if (filter === undefined) return;
+
+    const pages = Math.ceil(totalCount / filter.itemsPerPage);
     Mousetrap.bind("right", () => {
-      const maxPage = totalCount / filter.itemsPerPage;
+      if (filter.currentPage < pages) {
-      if (filter.currentPage < maxPage) {
         onChangePage(filter.currentPage + 1);
       }
     });
@@ -234,25 +247,18 @@ const RenderList = <
       }
     });
     Mousetrap.bind("shift+right", () => {
-      const maxPage = totalCount / filter.itemsPerPage + 1;
+      onChangePage(Math.min(pages, filter.currentPage + 10));
-      onChangePage(Math.min(maxPage, filter.currentPage + 10));
     });
     Mousetrap.bind("shift+left", () => {
       onChangePage(Math.max(1, filter.currentPage - 10));
     });
     Mousetrap.bind("ctrl+end", () => {
-      const maxPage = totalCount / filter.itemsPerPage + 1;
+      onChangePage(pages);
-      onChangePage(maxPage);
     });
     Mousetrap.bind("ctrl+home", () => {
       onChangePage(1);
     });
 
-    let unbindExtras: () => void;
-    if (addKeybinds) {
-      unbindExtras = addKeybinds(result, filter, selectedIds);
-    }
-
     return () => {
       Mousetrap.unbind("right");
       Mousetrap.unbind("left");
@@ -260,12 +266,22 @@ const RenderList = <
       Mousetrap.unbind("shift+left");
       Mousetrap.unbind("ctrl+end");
       Mousetrap.unbind("ctrl+home");
-
-      if (unbindExtras) {
-        unbindExtras();
-      }
     };
-  });
+  }, [filter, onChangePage, totalCount]);
+  useEffect(() => {
+    if (filter === undefined) return;
+
+    if (addKeybinds) {
+      const unbindExtras = addKeybinds(result, filter, selectedIds);
+      return () => {
+        unbindExtras();
+      };
+    }
+  }, [addKeybinds, filter, result, selectedIds]);
+
+  // Don't continue if filter is undefined
+  // There are no hooks below this point so this is valid
+  if (filter === undefined) return;
 
   function singleSelect(id: string, selected: boolean) {
     setLastClickedId(id);
@@ -334,24 +350,24 @@ const RenderList = <
     setLastClickedId(undefined);
   }
 
-  function onSelectNone() {
+  const onSelectNone = () => {
     const newSelectedIds: Set<string> = new Set();
     setSelectedIds(newSelectedIds);
     setLastClickedId(undefined);
-  }
+  };
 
-  function onChangeZoom(newZoomIndex: number) {
+  const onChangeZoom = (newZoomIndex: number) => {
     const newFilter = cloneDeep(filter);
     newFilter.zoomIndex = newZoomIndex;
     updateFilter(newFilter);
-  }
+  };
 
-  function onOperationClicked(o: IListHookOperation<QueryResult>) {
+  const onOperationClicked = async (o: IListHookOperation<QueryResult>) => {
-    o.onClick(result, filter, selectedIds);
+    await o.onClick(result, filter, selectedIds);
     if (o.postRefetch) {
       result.refetch();
     }
-  }
+  };
 
   const operations =
     otherOperations &&
@@ -409,11 +425,12 @@ const RenderList = <
     />
   );
 
-  function maybeRenderContent() {
+  const maybeRenderContent = () => {
     if (result.loading || result.error) {
       return;
     }
 
+    const pages = Math.ceil(totalCount / filter.itemsPerPage);
     return (
       <>
         {renderPagination()}
@@ -433,18 +450,18 @@ const RenderList = <
         {renderPagination()}
       </>
     );
-  }
+  };
 
-  function onChangeDisplayMode(displayMode: DisplayMode) {
+  const onChangeDisplayMode = (displayMode: DisplayMode) => {
     const newFilter = cloneDeep(filter);
     newFilter.displayMode = displayMode;
     updateFilter(newFilter);
-  }
+  };
 
-  function onAddCriterion(
+  const onAddCriterion = (
     criterion: Criterion<CriterionValue>,
     oldId?: string
-  ) {
+  ) => {
     const newFilter = cloneDeep(filter);
 
     // Find if we are editing an existing criteria, then modify that.  Or create a new one.
@@ -468,22 +485,22 @@ const RenderList = <
     updateFilter(newFilter);
     setEditingCriterion(undefined);
     setNewCriterion(false);
-  }
+  };
 
-  function onRemoveCriterion(removedCriterion: Criterion<CriterionValue>) {
+  const onRemoveCriterion = (removedCriterion: Criterion<CriterionValue>) => {
     const newFilter = cloneDeep(filter);
     newFilter.criteria = newFilter.criteria.filter(
       (criterion) => criterion.getId() !== removedCriterion.getId()
     );
     newFilter.currentPage = 1;
     updateFilter(newFilter);
-  }
+  };
 
-  function updateCriteria(c: Criterion<CriterionValue>[]) {
+  const updateCriteria = (c: Criterion<CriterionValue>[]) => {
     const newFilter = cloneDeep(filter);
     newFilter.criteria = c.slice();
     setNewCriterion(false);
-  }
+  };
 
   function onCancelAddCriterion() {
     setEditingCriterion(undefined);
@@ -732,10 +749,14 @@ const useList = <QueryResult extends IQueryResult, QueryData extends IDataItem>(
   );
 
   const renderFilter = useMemo(() => {
-    return !options.filterHook ? filter : options.filterHook(cloneDeep(filter));
+    if (filterInitialised) {
-  }, [filter, options]);
+      return options.filterHook
+        ? options.filterHook(cloneDeep(filter))
+        : filter;
+    }
+  }, [filterInitialised, filter, options]);
 
-  const { contentTemplate, onSelectChange } = RenderList({
+  const renderList = useRenderList({
     ...options,
     filter: renderFilter,
     filterOptions,
@@ -743,12 +764,18 @@ const useList = <QueryResult extends IQueryResult, QueryData extends IDataItem>(
     updateFilter,
   });
 
-  const template = !filterInitialised ? (
+  const template = renderList ? (
-    <LoadingIndicator />
+    renderList.contentTemplate
   ) : (
-    <>{contentTemplate}</>
+    <LoadingIndicator />
   );
 
+  function onSelectChange(id: string, selected: boolean, shiftKey: boolean) {
+    if (renderList) {
+      renderList.onSelectChange(id, selected, shiftKey);
+    }
+  }
+
   return {
     filter,
     template,