From 8e9d93f6f947dfa4422f95c184f97ef499a453ac Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Mon, 18 Jul 2022 11:54:56 +0800 Subject: [PATCH] perf: retrieve one to many collections in bulk --- .../komga/infrastructure/jooq/BookDtoDao.kt | 50 +++++++------ .../komga/infrastructure/jooq/SeriesDtoDao.kt | 70 +++++++++++-------- 2 files changed, 71 insertions(+), 49 deletions(-) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt index 73ab87d6d..a8f1aba88 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt @@ -27,12 +27,14 @@ import org.jooq.ResultQuery import org.jooq.impl.DSL import org.jooq.impl.DSL.inline import org.jooq.impl.DSL.noCondition +import org.springframework.beans.factory.annotation.Value import org.springframework.data.domain.Page import org.springframework.data.domain.PageImpl import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.stereotype.Component +import org.springframework.transaction.support.TransactionTemplate import java.math.BigDecimal import java.net.URL @@ -40,6 +42,8 @@ import java.net.URL class BookDtoDao( private val dsl: DSLContext, private val luceneHelper: LuceneHelper, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, + private val transactionTemplate: TransactionTemplate, ) : BookDtoRepository { private val b = Tables.BOOK @@ -298,8 +302,30 @@ class BookDtoDao( .apply { if (joinConditions.selectReadListNumber) leftJoin(rlb).on(b.ID.eq(rlb.BOOK_ID)) } .apply { if (joinConditions.author) leftJoin(a).on(b.ID.eq(a.BOOK_ID)) } - private fun ResultQuery.fetchAndMap() = - fetch() + private fun ResultQuery.fetchAndMap(): MutableList { + val records = fetch() + val bookIds = records.getValues(b.ID) + + lateinit var authors: Map> + lateinit var tags: Map> + lateinit var links: Map> + transactionTemplate.executeWithoutResult { + dsl.insertTempStrings(batchSize, bookIds) + authors = dsl.selectFrom(a) + .where(a.BOOK_ID.`in`(dsl.selectTempStrings())) + .filter { it.name != null } + .groupBy({ it.bookId }, { AuthorDto(it.name, it.role) }) + + tags = dsl.selectFrom(bt) + .where(bt.BOOK_ID.`in`(dsl.selectTempStrings())) + .groupBy({ it.bookId }, { it.tag }) + + links = dsl.selectFrom(bl) + .where(bl.BOOK_ID.`in`(dsl.selectTempStrings())) + .groupBy({ it.bookId }, { WebLinkDto(it.label, it.url) }) + } + + return records .map { rec -> val br = rec.into(b) val mr = rec.into(m) @@ -307,25 +333,9 @@ class BookDtoDao( val rr = rec.into(r) val seriesTitle = rec.into(sd.TITLE).component1() - val authors = dsl.selectFrom(a) - .where(a.BOOK_ID.eq(br.id)) - .fetchInto(a) - .filter { it.name != null } - .map { AuthorDto(it.name, it.role) } - - val tags = dsl.select(bt.TAG) - .from(bt) - .where(bt.BOOK_ID.eq(br.id)) - .fetchSet(bt.TAG) - - val links = dsl.select(bl.LABEL, bl.URL) - .from(bl) - .where(bl.BOOK_ID.eq(br.id)) - .fetchInto(bl) - .map { WebLinkDto(it.label, it.url) } - - br.toDto(mr.toDto(), dr.toDto(authors, tags, links), if (rr.userId != null) rr.toDto() else null, seriesTitle) + br.toDto(mr.toDto(), dr.toDto(authors[br.id].orEmpty(), tags[br.id].orEmpty().toSet(), links[br.id].orEmpty()), if (rr.userId != null) rr.toDto() else null, seriesTitle) } + } private fun BookSearchWithReadProgress.toCondition(): Condition { var c: Condition = noCondition() diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt index 554bbd3c1..d848019a6 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt @@ -29,12 +29,14 @@ import org.jooq.impl.DSL.count import org.jooq.impl.DSL.countDistinct import org.jooq.impl.DSL.lower import org.jooq.impl.DSL.substring +import org.springframework.beans.factory.annotation.Value import org.springframework.data.domain.Page import org.springframework.data.domain.PageImpl import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.stereotype.Component +import org.springframework.transaction.support.TransactionTemplate import java.net.URL private val logger = KotlinLogging.logger {} @@ -47,6 +49,8 @@ const val BOOKS_READ_COUNT = "booksReadCount" class SeriesDtoDao( private val dsl: DSLContext, private val luceneHelper: LuceneHelper, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, + private val transactionTemplate: TransactionTemplate, ) : SeriesDtoRepository { private val s = Tables.SERIES @@ -216,8 +220,40 @@ class SeriesDtoDao( private fun readProgressConditionSeries(userId: String): Condition = rs.USER_ID.eq(userId).or(rs.USER_ID.isNull) - private fun ResultQuery.fetchAndMap() = - fetch() + private fun ResultQuery.fetchAndMap(): MutableList { + val records = fetch() + val seriesIds = records.getValues(s.ID) + + lateinit var genres: Map> + lateinit var tags: Map> + lateinit var sharingLabels: Map> + lateinit var aggregatedAuthors: Map> + lateinit var aggregatedTags: Map> + transactionTemplate.executeWithoutResult { + dsl.insertTempStrings(batchSize, seriesIds) + genres = dsl.selectFrom(g) + .where(g.SERIES_ID.`in`(dsl.selectTempStrings())) + .groupBy({ it.seriesId }, { it.genre }) + + tags = dsl.selectFrom(st) + .where(st.SERIES_ID.`in`(dsl.selectTempStrings())) + .groupBy({ it.seriesId }, { it.tag }) + + sharingLabels = dsl.selectFrom(sl) + .where(sl.SERIES_ID.`in`(dsl.selectTempStrings())) + .groupBy({ it.seriesId }, { it.label }) + + aggregatedAuthors = dsl.selectFrom(bmaa) + .where(bmaa.SERIES_ID.`in`(dsl.selectTempStrings())) + .filter { it.name != null } + .groupBy({ it.seriesId }, { AuthorDto(it.name, it.role) }) + + aggregatedTags = dsl.selectFrom(bmat) + .where(bmat.SERIES_ID.`in`(dsl.selectTempStrings())) + .groupBy({ it.seriesId }, { it.tag }) + } + + return records .map { rec -> val sr = rec.into(s) val dr = rec.into(d) @@ -227,40 +263,16 @@ class SeriesDtoDao( val booksInProgressCount = rsr.inProgressCount ?: 0 val booksUnreadCount = sr.bookCount - booksReadCount - booksInProgressCount - val genres = dsl.select(g.GENRE) - .from(g) - .where(g.SERIES_ID.eq(sr.id)) - .fetchSet(g.GENRE) - - val tags = dsl.select(st.TAG) - .from(st) - .where(st.SERIES_ID.eq(sr.id)) - .fetchSet(st.TAG) - - val sharingLabels = dsl.select(sl.LABEL) - .from(sl) - .where(sl.SERIES_ID.eq(sr.id)) - .fetchSet(sl.LABEL) - - val aggregatedAuthors = dsl.selectFrom(bmaa) - .where(bmaa.SERIES_ID.eq(sr.id)) - .fetchInto(bmaa) - .filter { it.name != null } - .map { AuthorDto(it.name, it.role) } - - val aggregatedTags = dsl.selectFrom(bmat) - .where(bmat.SERIES_ID.eq(sr.id)) - .fetchSet(bmat.TAG) - sr.toDto( sr.bookCount, booksReadCount, booksUnreadCount, booksInProgressCount, - dr.toDto(genres, tags, sharingLabels), - bmar.toDto(aggregatedAuthors, aggregatedTags), + dr.toDto(genres[sr.id].orEmpty().toSet(), tags[sr.id].orEmpty().toSet(), sharingLabels[sr.id].orEmpty().toSet()), + bmar.toDto(aggregatedAuthors[sr.id].orEmpty(), aggregatedTags[sr.id].orEmpty().toSet()), ) } + } private fun SeriesSearchWithReadProgress.toCondition(): Condition { var c = DSL.noCondition()