Handle large and all entity queries (#3544)

* Remove upper page size limit
* Batch GetMany function
* Remove upper query limit from UI

pkg/models/find_filter.go

@@ -96,15 +96,15 @@ func (ff FindFilterType) GetPage() int {
 func (ff FindFilterType) GetPageSize() int {
 	const defaultPerPage = 25
 	const minPerPage = 0
-	const maxPerPage = 1000
 
 	if ff.PerPage == nil {
 		return defaultPerPage
 	}
 
-	if *ff.PerPage > maxPerPage {
+	// removed the maxPerPage check. We already all -1 to indicate all results
-		return maxPerPage
+	// so there is no conceivable reason we should limit the page size
-	} else if *ff.PerPage < minPerPage {
+
+	if *ff.PerPage < minPerPage {
 		// negative page sizes should return all results
 		// this is a sanity check in case GetPageSize is
 		// called with a negative page size.

pkg/sqlite/batch.go

@@ -0,0 +1,20 @@
+package sqlite
+
+const defaultBatchSize = 1000
+
+// batchExec executes the provided function in batches of the provided size.
+func batchExec(ids []int, batchSize int, fn func(batch []int) error) error {
+	for i := 0; i < len(ids); i += batchSize {
+		end := i + batchSize
+		if end > len(ids) {
+			end = len(ids)
+		}
+
+		batch := ids[i:end]
+		if err := fn(batch); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/sqlite/gallery.go

@@ -363,19 +363,25 @@ func (qb *GalleryStore) Find(ctx context.Context, id int) (*models.Gallery, erro
 }
 
 func (qb *GalleryStore) FindMany(ctx context.Context, ids []int) ([]*models.Gallery, error) {
-	q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(ids))
+	galleries := make([]*models.Gallery, len(ids))
+
+	if err := batchExec(ids, defaultBatchSize, func(batch []int) error {
+		q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(batch))
 		unsorted, err := qb.getMany(ctx, q)
 		if err != nil {
-		return nil, err
+			return err
 		}
 
-	galleries := make([]*models.Gallery, len(ids))
-
 		for _, s := range unsorted {
 			i := intslice.IntIndex(ids, s.ID)
 			galleries[i] = s
 		}
 
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
 	for i := range galleries {
 		if galleries[i] == nil {
 			return nil, fmt.Errorf("gallery with id %d not found", ids[i])

pkg/sqlite/image.go

@@ -260,19 +260,25 @@ func (qb *ImageStore) Find(ctx context.Context, id int) (*models.Image, error) {
 }
 
 func (qb *ImageStore) FindMany(ctx context.Context, ids []int) ([]*models.Image, error) {
-	q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(ids))
+	images := make([]*models.Image, len(ids))
+
+	if err := batchExec(ids, defaultBatchSize, func(batch []int) error {
+		q := qb.selectDataset().Prepared(true).Where(qb.table().Col(idColumn).In(batch))
 		unsorted, err := qb.getMany(ctx, q)
 		if err != nil {
-		return nil, err
+			return err
 		}
 
-	images := make([]*models.Image, len(ids))
-
 		for _, s := range unsorted {
 			i := intslice.IntIndex(ids, s.ID)
 			images[i] = s
 		}
 
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
 	for i := range images {
 		if images[i] == nil {
 			return nil, fmt.Errorf("image with id %d not found", ids[i])

pkg/sqlite/movies.go

@@ -70,19 +70,25 @@ func (qb *movieQueryBuilder) Find(ctx context.Context, id int) (*models.Movie, e
 
 func (qb *movieQueryBuilder) FindMany(ctx context.Context, ids []int) ([]*models.Movie, error) {
 	tableMgr := movieTableMgr
-	q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(ids...))
+	ret := make([]*models.Movie, len(ids))
+
+	if err := batchExec(ids, defaultBatchSize, func(batch []int) error {
+		q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(batch...))
 		unsorted, err := qb.getMany(ctx, q)
 		if err != nil {
-		return nil, err
+			return err
 		}
 
-	ret := make([]*models.Movie, len(ids))
-
 		for _, s := range unsorted {
 			i := intslice.IntIndex(ids, s.ID)
 			ret[i] = s
 		}
 
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
 	for i := range ret {
 		if ret[i] == nil {
 			return nil, fmt.Errorf("movie with id %d not found", ids[i])

pkg/sqlite/performer.go

@@ -299,19 +299,25 @@ func (qb *PerformerStore) Find(ctx context.Context, id int) (*models.Performer,
 
 func (qb *PerformerStore) FindMany(ctx context.Context, ids []int) ([]*models.Performer, error) {
 	tableMgr := performerTableMgr
-	q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(ids...))
+	ret := make([]*models.Performer, len(ids))
+
+	if err := batchExec(ids, defaultBatchSize, func(batch []int) error {
+		q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(batch...))
 		unsorted, err := qb.getMany(ctx, q)
 		if err != nil {
-		return nil, err
+			return err
 		}
 
-	ret := make([]*models.Performer, len(ids))
-
 		for _, s := range unsorted {
 			i := intslice.IntIndex(ids, s.ID)
 			ret[i] = s
 		}
 
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
 	for i := range ret {
 		if ret[i] == nil {
 			return nil, fmt.Errorf("performer with id %d not found", ids[i])

pkg/sqlite/scene.go

@@ -364,20 +364,26 @@ func (qb *SceneStore) Find(ctx context.Context, id int) (*models.Scene, error) {
 }
 
 func (qb *SceneStore) FindMany(ctx context.Context, ids []int) ([]*models.Scene, error) {
+	scenes := make([]*models.Scene, len(ids))
+
 	table := qb.table()
-	q := qb.selectDataset().Prepared(true).Where(table.Col(idColumn).In(ids))
+	if err := batchExec(ids, defaultBatchSize, func(batch []int) error {
+		q := qb.selectDataset().Prepared(true).Where(table.Col(idColumn).In(batch))
 		unsorted, err := qb.getMany(ctx, q)
 		if err != nil {
-		return nil, err
+			return err
 		}
 
-	scenes := make([]*models.Scene, len(ids))
-
 		for _, s := range unsorted {
 			i := intslice.IntIndex(ids, s.ID)
 			scenes[i] = s
 		}
 
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
 	for i := range scenes {
 		if scenes[i] == nil {
 			return nil, fmt.Errorf("scene with id %d not found", ids[i])

pkg/sqlite/studio.go

@@ -80,19 +80,25 @@ func (qb *studioQueryBuilder) Find(ctx context.Context, id int) (*models.Studio,
 
 func (qb *studioQueryBuilder) FindMany(ctx context.Context, ids []int) ([]*models.Studio, error) {
 	tableMgr := studioTableMgr
-	q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(ids...))
+	ret := make([]*models.Studio, len(ids))
+
+	if err := batchExec(ids, defaultBatchSize, func(batch []int) error {
+		q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(batch...))
 		unsorted, err := qb.getMany(ctx, q)
 		if err != nil {
-		return nil, err
+			return err
 		}
 
-	ret := make([]*models.Studio, len(ids))
-
 		for _, s := range unsorted {
 			i := intslice.IntIndex(ids, s.ID)
 			ret[i] = s
 		}
 
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
 	for i := range ret {
 		if ret[i] == nil {
 			return nil, fmt.Errorf("studio with id %d not found", ids[i])

pkg/sqlite/tag.go

@@ -98,19 +98,25 @@ func (qb *tagQueryBuilder) Find(ctx context.Context, id int) (*models.Tag, error
 
 func (qb *tagQueryBuilder) FindMany(ctx context.Context, ids []int) ([]*models.Tag, error) {
 	tableMgr := tagTableMgr
-	q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(ids...))
+	ret := make([]*models.Tag, len(ids))
+
+	if err := batchExec(ids, defaultBatchSize, func(batch []int) error {
+		q := goqu.Select("*").From(tableMgr.table).Where(tableMgr.byIDInts(batch...))
 		unsorted, err := qb.getMany(ctx, q)
 		if err != nil {
-		return nil, err
+			return err
 		}
 
-	ret := make([]*models.Tag, len(ids))
-
 		for _, s := range unsorted {
 			i := intslice.IntIndex(ids, s.ID)
 			ret[i] = s
 		}
 
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
 	for i := range ret {
 		if ret[i] == nil {
 			return nil, fmt.Errorf("tag with id %d not found", ids[i])

ui/v2.5/src/components/List/ListFilter.tsx

@@ -40,7 +40,6 @@ import {
 } from "@fortawesome/free-solid-svg-icons";
 import { useDebounce } from "src/hooks/debounce";
 
-const maxPageSize = 1000;
 interface IListFilterProps {
   onFilterUpdate: (newFilter: ListFilterModel) => void;
   filter: ListFilterModel;
@@ -134,11 +133,6 @@ export const ListFilter: React.FC<IListFilterProps> = ({
       return;
     }
 
-    // don't allow page sizes over 1000
-    if (pp > maxPageSize) {
-      pp = maxPageSize;
-    }
-
     const newFilter = cloneDeep(filter);
     newFilter.itemsPerPage = pp;
     newFilter.currentPage = 1;
@@ -377,7 +371,6 @@ export const ListFilter: React.FC<IListFilterProps> = ({
                   <Form.Control
                     type="number"
                     min={1}
-                    max={maxPageSize}
                     className="text-input"
                     ref={perPageInput}
                     onKeyPress={(e: React.KeyboardEvent<HTMLInputElement>) => {