From 1d0c57854c4b4ca63f66228193e64bcf8af2c091 Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Sat, 11 Sep 2021 10:24:31 +0800 Subject: [PATCH] fix: insert batch in chunks closes #654 --- .../configuration/KomgaProperties.kt | 4 +- .../komga/infrastructure/jooq/BookDao.kt | 83 ++++++------ .../jooq/BookMetadataAggregationDao.kt | 41 +++--- .../infrastructure/jooq/BookMetadataDao.kt | 126 ++++++++++-------- .../komga/infrastructure/jooq/MediaDao.kt | 126 ++++++++++-------- .../infrastructure/jooq/ReadProgressDao.kt | 6 +- .../komga/infrastructure/jooq/SeriesDao.kt | 20 +-- .../infrastructure/jooq/SeriesMetadataDao.kt | 40 +++--- .../komga/infrastructure/jooq/SidecarDao.kt | 21 +-- 9 files changed, 254 insertions(+), 213 deletions(-) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt index 6de7dab50..342a834e3 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt @@ -48,8 +48,8 @@ class KomgaProperties { @get:NotBlank var file: String = "" - @Deprecated("Unused since 0.81.0") - var batchSize: Int = 500 + @get:Positive + var batchChunkSize: Int = 1000 } class Lucene { diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt index 26a4ad9ee..edf125a1c 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt @@ -8,6 +8,7 @@ import org.gotson.komga.jooq.tables.records.BookRecord import org.jooq.Condition import org.jooq.DSLContext import org.jooq.impl.DSL +import org.springframework.beans.factory.annotation.Value import org.springframework.data.domain.Page import org.springframework.data.domain.PageImpl import org.springframework.data.domain.PageRequest @@ -21,9 +22,9 @@ import java.time.ZoneId @Component class BookDao( - private val dsl: DSLContext + private val dsl: DSLContext, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : BookRepository { - private val b = Tables.BOOK private val m = Tables.MEDIA private val d = Tables.BOOK_METADATA @@ -70,13 +71,15 @@ class BookDao( dsl.deleteFrom(u).execute() if (urls.isNotEmpty()) { - dsl.batch( - dsl.insertInto(u, u.URL).values(null as String?) - ).also { step -> - urls.forEach { - step.bind(it.toString()) - } - }.execute() + urls.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto(u, u.URL).values(null as String?) + ).also { step -> + chunk.forEach { + step.bind(it.toString()) + } + }.execute() + } } return dsl.selectFrom(b) @@ -243,36 +246,38 @@ class BookDao( @Transactional override fun insert(books: Collection) { if (books.isNotEmpty()) { - dsl.batch( - dsl.insertInto( - b, - b.ID, - b.NAME, - b.URL, - b.NUMBER, - b.FILE_LAST_MODIFIED, - b.FILE_SIZE, - b.FILE_HASH, - b.LIBRARY_ID, - b.SERIES_ID, - b.DELETED_DATE, - ).values(null as String?, null, null, null, null, null, null, null, null, null) - ).also { step -> - books.forEach { - step.bind( - it.id, - it.name, - it.url, - it.number, - it.fileLastModified, - it.fileSize, - it.fileHash, - it.libraryId, - it.seriesId, - it.deletedDate, - ) - } - }.execute() + books.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto( + b, + b.ID, + b.NAME, + b.URL, + b.NUMBER, + b.FILE_LAST_MODIFIED, + b.FILE_SIZE, + b.FILE_HASH, + b.LIBRARY_ID, + b.SERIES_ID, + b.DELETED_DATE, + ).values(null as String?, null, null, null, null, null, null, null, null, null) + ).also { step -> + chunk.forEach { + step.bind( + it.id, + it.name, + it.url, + it.number, + it.fileLastModified, + it.fileSize, + it.fileHash, + it.libraryId, + it.seriesId, + it.deletedDate, + ) + } + }.execute() + } } } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt index 62734be0e..16624e0ce 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt @@ -7,6 +7,7 @@ import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.BookMetadataAggregationAuthorRecord import org.gotson.komga.jooq.tables.records.BookMetadataAggregationRecord import org.jooq.DSLContext +import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime @@ -14,9 +15,9 @@ import java.time.ZoneId @Component class BookMetadataAggregationDao( - private val dsl: DSLContext + private val dsl: DSLContext, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : BookMetadataAggregationRepository { - private val d = Tables.BOOK_METADATA_AGGREGATION private val a = Tables.BOOK_METADATA_AGGREGATION_AUTHOR private val t = Tables.BOOK_METADATA_AGGREGATION_TAG @@ -83,27 +84,31 @@ class BookMetadataAggregationDao( private fun insertAuthors(metadata: BookMetadataAggregation) { if (metadata.authors.isNotEmpty()) { - dsl.batch( - dsl.insertInto(a, a.SERIES_ID, a.NAME, a.ROLE) - .values(null as String?, null, null) - ).also { step -> - metadata.authors.forEach { - step.bind(metadata.seriesId, it.name, it.role) - } - }.execute() + metadata.authors.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto(a, a.SERIES_ID, a.NAME, a.ROLE) + .values(null as String?, null, null) + ).also { step -> + chunk.forEach { + step.bind(metadata.seriesId, it.name, it.role) + } + }.execute() + } } } private fun insertTags(metadata: BookMetadataAggregation) { if (metadata.tags.isNotEmpty()) { - dsl.batch( - dsl.insertInto(t, t.SERIES_ID, t.TAG) - .values(null as String?, null) - ).also { step -> - metadata.tags.forEach { - step.bind(metadata.seriesId, it) - } - }.execute() + metadata.tags.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto(t, t.SERIES_ID, t.TAG) + .values(null as String?, null) + ).also { step -> + chunk.forEach { + step.bind(metadata.seriesId, it) + } + }.execute() + } } } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt index 52dcc9f96..c8f5825ef 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt @@ -7,6 +7,7 @@ import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.BookMetadataAuthorRecord import org.gotson.komga.jooq.tables.records.BookMetadataRecord import org.jooq.DSLContext +import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime @@ -14,7 +15,8 @@ import java.time.ZoneId @Component class BookMetadataDao( - private val dsl: DSLContext + private val dsl: DSLContext, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : BookMetadataRepository { private val d = Tables.BOOK_METADATA @@ -60,46 +62,48 @@ class BookMetadataDao( @Transactional override fun insert(metadatas: Collection) { if (metadatas.isNotEmpty()) { - dsl.batch( - dsl.insertInto( - d, - d.BOOK_ID, - d.TITLE, - d.TITLE_LOCK, - d.SUMMARY, - d.SUMMARY_LOCK, - d.NUMBER, - d.NUMBER_LOCK, - d.NUMBER_SORT, - d.NUMBER_SORT_LOCK, - d.RELEASE_DATE, - d.RELEASE_DATE_LOCK, - d.AUTHORS_LOCK, - d.TAGS_LOCK, - d.ISBN, - d.ISBN_LOCK - ).values(null as String?, null, null, null, null, null, null, null, null, null, null, null, null, null, null) - ).also { step -> - metadatas.forEach { - step.bind( - it.bookId, - it.title, - it.titleLock, - it.summary, - it.summaryLock, - it.number, - it.numberLock, - it.numberSort, - it.numberSortLock, - it.releaseDate, - it.releaseDateLock, - it.authorsLock, - it.tagsLock, - it.isbn, - it.isbnLock - ) - } - }.execute() + metadatas.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto( + d, + d.BOOK_ID, + d.TITLE, + d.TITLE_LOCK, + d.SUMMARY, + d.SUMMARY_LOCK, + d.NUMBER, + d.NUMBER_LOCK, + d.NUMBER_SORT, + d.NUMBER_SORT_LOCK, + d.RELEASE_DATE, + d.RELEASE_DATE_LOCK, + d.AUTHORS_LOCK, + d.TAGS_LOCK, + d.ISBN, + d.ISBN_LOCK + ).values(null as String?, null, null, null, null, null, null, null, null, null, null, null, null, null, null) + ).also { step -> + chunk.forEach { + step.bind( + it.bookId, + it.title, + it.titleLock, + it.summary, + it.summaryLock, + it.number, + it.numberLock, + it.numberSort, + it.numberSortLock, + it.releaseDate, + it.releaseDateLock, + it.authorsLock, + it.tagsLock, + it.isbn, + it.isbnLock + ) + } + }.execute() + } insertAuthors(metadatas) insertTags(metadatas) @@ -149,31 +153,35 @@ class BookMetadataDao( private fun insertAuthors(metadatas: Collection) { if (metadatas.any { it.authors.isNotEmpty() }) { - dsl.batch( - dsl.insertInto(a, a.BOOK_ID, a.NAME, a.ROLE) - .values(null as String?, null, null) - ).also { step -> - metadatas.forEach { metadata -> - metadata.authors.forEach { - step.bind(metadata.bookId, it.name, it.role) + metadatas.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto(a, a.BOOK_ID, a.NAME, a.ROLE) + .values(null as String?, null, null) + ).also { step -> + chunk.forEach { metadata -> + metadata.authors.forEach { + step.bind(metadata.bookId, it.name, it.role) + } } - } - }.execute() + }.execute() + } } } private fun insertTags(metadatas: Collection) { if (metadatas.any { it.tags.isNotEmpty() }) { - dsl.batch( - dsl.insertInto(bt, bt.BOOK_ID, bt.TAG) - .values(null as String?, null) - ).also { step -> - metadatas.forEach { metadata -> - metadata.tags.forEach { - step.bind(metadata.bookId, it) + metadatas.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto(bt, bt.BOOK_ID, bt.TAG) + .values(null as String?, null) + ).also { step -> + chunk.forEach { metadata -> + metadata.tags.forEach { + step.bind(metadata.bookId, it) + } } - } - }.execute() + }.execute() + } } } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt index 2d9ff2a8a..bf685b0cb 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt @@ -8,6 +8,7 @@ import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.MediaPageRecord import org.gotson.komga.jooq.tables.records.MediaRecord import org.jooq.DSLContext +import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime @@ -15,7 +16,8 @@ import java.time.ZoneId @Component class MediaDao( - private val dsl: DSLContext + private val dsl: DSLContext, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : MediaRepository { private val m = Tables.MEDIA @@ -67,26 +69,28 @@ class MediaDao( @Transactional override fun insert(medias: Collection) { if (medias.isNotEmpty()) { - dsl.batch( - dsl.insertInto( - m, - m.BOOK_ID, - m.STATUS, - m.MEDIA_TYPE, - m.COMMENT, - m.PAGE_COUNT - ).values(null as String?, null, null, null, null) - ).also { step -> - medias.forEach { - step.bind( - it.bookId, - it.status, - it.mediaType, - it.comment, - it.pages.size - ) - } - }.execute() + medias.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto( + m, + m.BOOK_ID, + m.STATUS, + m.MEDIA_TYPE, + m.COMMENT, + m.PAGE_COUNT + ).values(null as String?, null, null, null, null) + ).also { step -> + chunk.forEach { + step.bind( + it.bookId, + it.status, + it.mediaType, + it.comment, + it.pages.size + ) + } + }.execute() + } insertPages(medias) insertFiles(medias) @@ -95,51 +99,55 @@ class MediaDao( private fun insertPages(medias: Collection) { if (medias.any { it.pages.isNotEmpty() }) { - dsl.batch( - dsl.insertInto( - p, - p.BOOK_ID, - p.FILE_NAME, - p.MEDIA_TYPE, - p.NUMBER, - p.WIDTH, - p.HEIGHT - ).values(null as String?, null, null, null, null, null) - ).also { - medias.forEach { media -> - media.pages.forEachIndexed { index, page -> - it.bind( - media.bookId, - page.fileName, - page.mediaType, - index, - page.dimension?.width, - page.dimension?.height - ) + medias.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto( + p, + p.BOOK_ID, + p.FILE_NAME, + p.MEDIA_TYPE, + p.NUMBER, + p.WIDTH, + p.HEIGHT + ).values(null as String?, null, null, null, null, null) + ).also { step -> + chunk.forEach { media -> + media.pages.forEachIndexed { index, page -> + step.bind( + media.bookId, + page.fileName, + page.mediaType, + index, + page.dimension?.width, + page.dimension?.height + ) + } } - } - }.execute() + }.execute() + } } } private fun insertFiles(medias: Collection) { if (medias.any { it.files.isNotEmpty() }) { - dsl.batch( - dsl.insertInto( - f, - f.BOOK_ID, - f.FILE_NAME - ).values(null as String?, null) - ).also { step -> - medias.forEach { media -> - media.files.forEach { - step.bind( - media.bookId, - it - ) + medias.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto( + f, + f.BOOK_ID, + f.FILE_NAME + ).values(null as String?, null) + ).also { step -> + chunk.forEach { media -> + media.files.forEach { + step.bind( + media.bookId, + it + ) + } } - } - }.execute() + }.execute() + } } } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt index 5ca1d1b79..2464031da 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt @@ -8,6 +8,7 @@ import org.gotson.komga.jooq.tables.records.ReadProgressRecord import org.jooq.DSLContext import org.jooq.Query import org.jooq.impl.DSL +import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime @@ -15,7 +16,8 @@ import java.time.ZoneId @Component class ReadProgressDao( - private val dsl: DSLContext + private val dsl: DSLContext, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : ReadProgressRepository { private val r = Tables.READ_PROGRESS @@ -60,7 +62,7 @@ class ReadProgressDao( @Transactional override fun save(readProgresses: Collection) { val queries = readProgresses.map { saveQuery(it) } - dsl.batch(queries).execute() + queries.chunked(batchSize).forEach { chunk -> dsl.batch(chunk).execute() } readProgresses.groupBy { it.userId } .forEach { (userId, readProgresses) -> diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt index dc957e997..0e0dd08b1 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt @@ -8,6 +8,7 @@ import org.gotson.komga.jooq.tables.records.SeriesRecord import org.jooq.Condition import org.jooq.DSLContext import org.jooq.impl.DSL +import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.net.URL @@ -16,9 +17,9 @@ import java.time.ZoneId @Component class SeriesDao( - private val dsl: DSLContext + private val dsl: DSLContext, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : SeriesRepository { - private val s = Tables.SERIES private val d = Tables.SERIES_METADATA private val cs = Tables.COLLECTION_SERIES @@ -47,13 +48,16 @@ class SeriesDao( dsl.deleteFrom(u).execute() if (urls.isNotEmpty()) { - dsl.batch( - dsl.insertInto(u, u.URL).values(null as String?) - ).also { step -> - urls.forEach { - step.bind(it.toString()) + urls.chunked(batchSize) + .forEach { chunk -> + dsl.batch( + dsl.insertInto(u, u.URL).values(null as String?) + ).also { step -> + chunk.forEach { + step.bind(it.toString()) + } + }.execute() } - }.execute() } return dsl.selectFrom(s) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt index 5d72dac9f..f1683744d 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt @@ -5,6 +5,7 @@ import org.gotson.komga.domain.persistence.SeriesMetadataRepository import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.SeriesMetadataRecord import org.jooq.DSLContext +import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.time.LocalDateTime @@ -12,7 +13,8 @@ import java.time.ZoneId @Component class SeriesMetadataDao( - private val dsl: DSLContext + private val dsl: DSLContext, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : SeriesMetadataRepository { private val d = Tables.SERIES_METADATA @@ -117,27 +119,31 @@ class SeriesMetadataDao( private fun insertGenres(metadata: SeriesMetadata) { if (metadata.genres.isNotEmpty()) { - dsl.batch( - dsl.insertInto(g, g.SERIES_ID, g.GENRE) - .values(null as String?, null) - ).also { step -> - metadata.genres.forEach { - step.bind(metadata.seriesId, it) - } - }.execute() + metadata.genres.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto(g, g.SERIES_ID, g.GENRE) + .values(null as String?, null) + ).also { step -> + chunk.forEach { + step.bind(metadata.seriesId, it) + } + }.execute() + } } } private fun insertTags(metadata: SeriesMetadata) { if (metadata.tags.isNotEmpty()) { - dsl.batch( - dsl.insertInto(st, st.SERIES_ID, st.TAG) - .values(null as String?, null) - ).also { step -> - metadata.tags.forEach { - step.bind(metadata.seriesId, it) - } - }.execute() + metadata.tags.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto(st, st.SERIES_ID, st.TAG) + .values(null as String?, null) + ).also { step -> + chunk.forEach { + step.bind(metadata.seriesId, it) + } + }.execute() + } } } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt index bfd61cda4..c170666c9 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt @@ -6,15 +6,16 @@ import org.gotson.komga.domain.persistence.SidecarRepository import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.SidecarRecord import org.jooq.DSLContext +import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.net.URL @Component class SidecarDao( - private val dsl: DSLContext + private val dsl: DSLContext, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : SidecarRepository { - private val sc = Tables.SIDECAR private val u = Tables.TEMP_URL_LIST @@ -42,13 +43,15 @@ class SidecarDao( dsl.deleteFrom(u).execute() if (urls.isNotEmpty()) { - dsl.batch( - dsl.insertInto(u, u.URL).values(null as String?) - ).also { step -> - urls.forEach { - step.bind(it.toString()) - } - }.execute() + urls.chunked(batchSize).forEach { chunk -> + dsl.batch( + dsl.insertInto(u, u.URL).values(null as String?) + ).also { step -> + chunk.forEach { + step.bind(it.toString()) + } + }.execute() + } } dsl.deleteFrom(sc)