fix: insert batch in chunks

closes #654
2025-12-20 15:34:17 +01:00 · 2021-09-11 10:24:31 +08:00 · 2021-09-11 10:24:31 +08:00 · 1d0c57854c
commit 1d0c57854c
parent 72d3451140
9 changed files with 254 additions and 213 deletions
--- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt
+++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt
@ -48,8 +48,8 @@ class KomgaProperties {
    @get:NotBlank
    var file: String = ""

-    @Deprecated("Unused since 0.81.0")
-    var batchSize: Int = 500
+    @get:Positive
+    var batchChunkSize: Int = 1000
  }

  class Lucene {
--- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt
+++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt
@ -8,6 +8,7 @@ import org.gotson.komga.jooq.tables.records.BookRecord
 import org.jooq.Condition
 import org.jooq.DSLContext
 import org.jooq.impl.DSL
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.data.domain.Page
 import org.springframework.data.domain.PageImpl
 import org.springframework.data.domain.PageRequest
@ -21,9 +22,9 @@ import java.time.ZoneId

@Component
 class BookDao(
-  private val dsl: DSLContext
+  private val dsl: DSLContext,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : BookRepository {
-
  private val b = Tables.BOOK
  private val m = Tables.MEDIA
  private val d = Tables.BOOK_METADATA
@ -70,13 +71,15 @@ class BookDao(
    dsl.deleteFrom(u).execute()

    if (urls.isNotEmpty()) {
-      dsl.batch(
-        dsl.insertInto(u, u.URL).values(null as String?)
-      ).also { step ->
-        urls.forEach {
-          step.bind(it.toString())
-        }
-      }.execute()
+      urls.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(u, u.URL).values(null as String?)
+        ).also { step ->
+          chunk.forEach {
+            step.bind(it.toString())
+          }
+        }.execute()
+      }
    }

    return dsl.selectFrom(b)
@ -243,36 +246,38 @@ class BookDao(
  @Transactional
  override fun insert(books: Collection<Book>) {
    if (books.isNotEmpty()) {
-      dsl.batch(
-        dsl.insertInto(
-          b,
-          b.ID,
-          b.NAME,
-          b.URL,
-          b.NUMBER,
-          b.FILE_LAST_MODIFIED,
-          b.FILE_SIZE,
-          b.FILE_HASH,
-          b.LIBRARY_ID,
-          b.SERIES_ID,
-          b.DELETED_DATE,
-        ).values(null as String?, null, null, null, null, null, null, null, null, null)
-      ).also { step ->
-        books.forEach {
-          step.bind(
-            it.id,
-            it.name,
-            it.url,
-            it.number,
-            it.fileLastModified,
-            it.fileSize,
-            it.fileHash,
-            it.libraryId,
-            it.seriesId,
-            it.deletedDate,
-          )
-        }
-      }.execute()
+      books.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(
+            b,
+            b.ID,
+            b.NAME,
+            b.URL,
+            b.NUMBER,
+            b.FILE_LAST_MODIFIED,
+            b.FILE_SIZE,
+            b.FILE_HASH,
+            b.LIBRARY_ID,
+            b.SERIES_ID,
+            b.DELETED_DATE,
+          ).values(null as String?, null, null, null, null, null, null, null, null, null)
+        ).also { step ->
+          chunk.forEach {
+            step.bind(
+              it.id,
+              it.name,
+              it.url,
+              it.number,
+              it.fileLastModified,
+              it.fileSize,
+              it.fileHash,
+              it.libraryId,
+              it.seriesId,
+              it.deletedDate,
+            )
+          }
+        }.execute()
+      }
    }
  }

--- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt
+++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt
@ -7,6 +7,7 @@ import org.gotson.komga.jooq.Tables
 import org.gotson.komga.jooq.tables.records.BookMetadataAggregationAuthorRecord
 import org.gotson.komga.jooq.tables.records.BookMetadataAggregationRecord
 import org.jooq.DSLContext
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 import java.time.LocalDateTime
@ -14,9 +15,9 @@ import java.time.ZoneId

@Component
 class BookMetadataAggregationDao(
-  private val dsl: DSLContext
+  private val dsl: DSLContext,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : BookMetadataAggregationRepository {
-
  private val d = Tables.BOOK_METADATA_AGGREGATION
  private val a = Tables.BOOK_METADATA_AGGREGATION_AUTHOR
  private val t = Tables.BOOK_METADATA_AGGREGATION_TAG
@ -83,27 +84,31 @@ class BookMetadataAggregationDao(

  private fun insertAuthors(metadata: BookMetadataAggregation) {
    if (metadata.authors.isNotEmpty()) {
-      dsl.batch(
-        dsl.insertInto(a, a.SERIES_ID, a.NAME, a.ROLE)
-          .values(null as String?, null, null)
-      ).also { step ->
-        metadata.authors.forEach {
-          step.bind(metadata.seriesId, it.name, it.role)
-        }
-      }.execute()
+      metadata.authors.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(a, a.SERIES_ID, a.NAME, a.ROLE)
+            .values(null as String?, null, null)
+        ).also { step ->
+          chunk.forEach {
+            step.bind(metadata.seriesId, it.name, it.role)
+          }
+        }.execute()
+      }
    }
  }

  private fun insertTags(metadata: BookMetadataAggregation) {
    if (metadata.tags.isNotEmpty()) {
-      dsl.batch(
-        dsl.insertInto(t, t.SERIES_ID, t.TAG)
-          .values(null as String?, null)
-      ).also { step ->
-        metadata.tags.forEach {
-          step.bind(metadata.seriesId, it)
-        }
-      }.execute()
+      metadata.tags.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(t, t.SERIES_ID, t.TAG)
+            .values(null as String?, null)
+        ).also { step ->
+          chunk.forEach {
+            step.bind(metadata.seriesId, it)
+          }
+        }.execute()
+      }
    }
  }

--- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt
+++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt
@ -7,6 +7,7 @@ import org.gotson.komga.jooq.Tables
 import org.gotson.komga.jooq.tables.records.BookMetadataAuthorRecord
 import org.gotson.komga.jooq.tables.records.BookMetadataRecord
 import org.jooq.DSLContext
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 import java.time.LocalDateTime
@ -14,7 +15,8 @@ import java.time.ZoneId

@Component
 class BookMetadataDao(
-  private val dsl: DSLContext
+  private val dsl: DSLContext,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : BookMetadataRepository {

  private val d = Tables.BOOK_METADATA
@ -60,46 +62,48 @@ class BookMetadataDao(
  @Transactional
  override fun insert(metadatas: Collection<BookMetadata>) {
    if (metadatas.isNotEmpty()) {
-      dsl.batch(
-        dsl.insertInto(
-          d,
-          d.BOOK_ID,
-          d.TITLE,
-          d.TITLE_LOCK,
-          d.SUMMARY,
-          d.SUMMARY_LOCK,
-          d.NUMBER,
-          d.NUMBER_LOCK,
-          d.NUMBER_SORT,
-          d.NUMBER_SORT_LOCK,
-          d.RELEASE_DATE,
-          d.RELEASE_DATE_LOCK,
-          d.AUTHORS_LOCK,
-          d.TAGS_LOCK,
-          d.ISBN,
-          d.ISBN_LOCK
-        ).values(null as String?, null, null, null, null, null, null, null, null, null, null, null, null, null, null)
-      ).also { step ->
-        metadatas.forEach {
-          step.bind(
-            it.bookId,
-            it.title,
-            it.titleLock,
-            it.summary,
-            it.summaryLock,
-            it.number,
-            it.numberLock,
-            it.numberSort,
-            it.numberSortLock,
-            it.releaseDate,
-            it.releaseDateLock,
-            it.authorsLock,
-            it.tagsLock,
-            it.isbn,
-            it.isbnLock
-          )
-        }
-      }.execute()
+      metadatas.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(
+            d,
+            d.BOOK_ID,
+            d.TITLE,
+            d.TITLE_LOCK,
+            d.SUMMARY,
+            d.SUMMARY_LOCK,
+            d.NUMBER,
+            d.NUMBER_LOCK,
+            d.NUMBER_SORT,
+            d.NUMBER_SORT_LOCK,
+            d.RELEASE_DATE,
+            d.RELEASE_DATE_LOCK,
+            d.AUTHORS_LOCK,
+            d.TAGS_LOCK,
+            d.ISBN,
+            d.ISBN_LOCK
+          ).values(null as String?, null, null, null, null, null, null, null, null, null, null, null, null, null, null)
+        ).also { step ->
+          chunk.forEach {
+            step.bind(
+              it.bookId,
+              it.title,
+              it.titleLock,
+              it.summary,
+              it.summaryLock,
+              it.number,
+              it.numberLock,
+              it.numberSort,
+              it.numberSortLock,
+              it.releaseDate,
+              it.releaseDateLock,
+              it.authorsLock,
+              it.tagsLock,
+              it.isbn,
+              it.isbnLock
+            )
+          }
+        }.execute()
+      }

      insertAuthors(metadatas)
      insertTags(metadatas)
@ -149,31 +153,35 @@ class BookMetadataDao(

  private fun insertAuthors(metadatas: Collection<BookMetadata>) {
    if (metadatas.any { it.authors.isNotEmpty() }) {
-      dsl.batch(
-        dsl.insertInto(a, a.BOOK_ID, a.NAME, a.ROLE)
-          .values(null as String?, null, null)
-      ).also { step ->
-        metadatas.forEach { metadata ->
-          metadata.authors.forEach {
-            step.bind(metadata.bookId, it.name, it.role)
+      metadatas.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(a, a.BOOK_ID, a.NAME, a.ROLE)
+            .values(null as String?, null, null)
+        ).also { step ->
+          chunk.forEach { metadata ->
+            metadata.authors.forEach {
+              step.bind(metadata.bookId, it.name, it.role)
+            }
          }
-        }
-      }.execute()
+        }.execute()
+      }
    }
  }

  private fun insertTags(metadatas: Collection<BookMetadata>) {
    if (metadatas.any { it.tags.isNotEmpty() }) {
-      dsl.batch(
-        dsl.insertInto(bt, bt.BOOK_ID, bt.TAG)
-          .values(null as String?, null)
-      ).also { step ->
-        metadatas.forEach { metadata ->
-          metadata.tags.forEach {
-            step.bind(metadata.bookId, it)
+      metadatas.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(bt, bt.BOOK_ID, bt.TAG)
+            .values(null as String?, null)
+        ).also { step ->
+          chunk.forEach { metadata ->
+            metadata.tags.forEach {
+              step.bind(metadata.bookId, it)
+            }
          }
-        }
-      }.execute()
+        }.execute()
+      }
    }
  }

--- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt
+++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt
@ -8,6 +8,7 @@ import org.gotson.komga.jooq.Tables
 import org.gotson.komga.jooq.tables.records.MediaPageRecord
 import org.gotson.komga.jooq.tables.records.MediaRecord
 import org.jooq.DSLContext
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 import java.time.LocalDateTime
@ -15,7 +16,8 @@ import java.time.ZoneId

@Component
 class MediaDao(
-  private val dsl: DSLContext
+  private val dsl: DSLContext,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : MediaRepository {

  private val m = Tables.MEDIA
@ -67,26 +69,28 @@ class MediaDao(
  @Transactional
  override fun insert(medias: Collection<Media>) {
    if (medias.isNotEmpty()) {
-      dsl.batch(
-        dsl.insertInto(
-          m,
-          m.BOOK_ID,
-          m.STATUS,
-          m.MEDIA_TYPE,
-          m.COMMENT,
-          m.PAGE_COUNT
-        ).values(null as String?, null, null, null, null)
-      ).also { step ->
-        medias.forEach {
-          step.bind(
-            it.bookId,
-            it.status,
-            it.mediaType,
-            it.comment,
-            it.pages.size
-          )
-        }
-      }.execute()
+      medias.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(
+            m,
+            m.BOOK_ID,
+            m.STATUS,
+            m.MEDIA_TYPE,
+            m.COMMENT,
+            m.PAGE_COUNT
+          ).values(null as String?, null, null, null, null)
+        ).also { step ->
+          chunk.forEach {
+            step.bind(
+              it.bookId,
+              it.status,
+              it.mediaType,
+              it.comment,
+              it.pages.size
+            )
+          }
+        }.execute()
+      }

      insertPages(medias)
      insertFiles(medias)
@ -95,51 +99,55 @@ class MediaDao(

  private fun insertPages(medias: Collection<Media>) {
    if (medias.any { it.pages.isNotEmpty() }) {
-      dsl.batch(
-        dsl.insertInto(
-          p,
-          p.BOOK_ID,
-          p.FILE_NAME,
-          p.MEDIA_TYPE,
-          p.NUMBER,
-          p.WIDTH,
-          p.HEIGHT
-        ).values(null as String?, null, null, null, null, null)
-      ).also {
-        medias.forEach { media ->
-          media.pages.forEachIndexed { index, page ->
-            it.bind(
-              media.bookId,
-              page.fileName,
-              page.mediaType,
-              index,
-              page.dimension?.width,
-              page.dimension?.height
-            )
+      medias.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(
+            p,
+            p.BOOK_ID,
+            p.FILE_NAME,
+            p.MEDIA_TYPE,
+            p.NUMBER,
+            p.WIDTH,
+            p.HEIGHT
+          ).values(null as String?, null, null, null, null, null)
+        ).also { step ->
+          chunk.forEach { media ->
+            media.pages.forEachIndexed { index, page ->
+              step.bind(
+                media.bookId,
+                page.fileName,
+                page.mediaType,
+                index,
+                page.dimension?.width,
+                page.dimension?.height
+              )
+            }
          }
-        }
-      }.execute()
+        }.execute()
+      }
    }
  }

  private fun insertFiles(medias: Collection<Media>) {
    if (medias.any { it.files.isNotEmpty() }) {
-      dsl.batch(
-        dsl.insertInto(
-          f,
-          f.BOOK_ID,
-          f.FILE_NAME
-        ).values(null as String?, null)
-      ).also { step ->
-        medias.forEach { media ->
-          media.files.forEach {
-            step.bind(
-              media.bookId,
-              it
-            )
+      medias.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(
+            f,
+            f.BOOK_ID,
+            f.FILE_NAME
+          ).values(null as String?, null)
+        ).also { step ->
+          chunk.forEach { media ->
+            media.files.forEach {
+              step.bind(
+                media.bookId,
+                it
+              )
+            }
          }
-        }
-      }.execute()
+        }.execute()
+      }
    }
  }

--- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt
+++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt
@ -8,6 +8,7 @@ import org.gotson.komga.jooq.tables.records.ReadProgressRecord
 import org.jooq.DSLContext
 import org.jooq.Query
 import org.jooq.impl.DSL
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 import java.time.LocalDateTime
@ -15,7 +16,8 @@ import java.time.ZoneId

@Component
 class ReadProgressDao(
-  private val dsl: DSLContext
+  private val dsl: DSLContext,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : ReadProgressRepository {

  private val r = Tables.READ_PROGRESS
@ -60,7 +62,7 @@ class ReadProgressDao(
  @Transactional
  override fun save(readProgresses: Collection<ReadProgress>) {
    val queries = readProgresses.map { saveQuery(it) }
-    dsl.batch(queries).execute()
+    queries.chunked(batchSize).forEach { chunk -> dsl.batch(chunk).execute() }

    readProgresses.groupBy { it.userId }
      .forEach { (userId, readProgresses) ->
--- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt
+++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt
@ -8,6 +8,7 @@ import org.gotson.komga.jooq.tables.records.SeriesRecord
 import org.jooq.Condition
 import org.jooq.DSLContext
 import org.jooq.impl.DSL
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 import java.net.URL
@ -16,9 +17,9 @@ import java.time.ZoneId

@Component
 class SeriesDao(
-  private val dsl: DSLContext
+  private val dsl: DSLContext,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : SeriesRepository {
-
  private val s = Tables.SERIES
  private val d = Tables.SERIES_METADATA
  private val cs = Tables.COLLECTION_SERIES
@ -47,13 +48,16 @@ class SeriesDao(
    dsl.deleteFrom(u).execute()

    if (urls.isNotEmpty()) {
-      dsl.batch(
-        dsl.insertInto(u, u.URL).values(null as String?)
-      ).also { step ->
-        urls.forEach {
-          step.bind(it.toString())
+      urls.chunked(batchSize)
+        .forEach { chunk ->
+          dsl.batch(
+            dsl.insertInto(u, u.URL).values(null as String?)
+          ).also { step ->
+            chunk.forEach {
+              step.bind(it.toString())
+            }
+          }.execute()
        }
-      }.execute()
    }

    return dsl.selectFrom(s)
--- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt
+++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt
@ -5,6 +5,7 @@ import org.gotson.komga.domain.persistence.SeriesMetadataRepository
 import org.gotson.komga.jooq.Tables
 import org.gotson.komga.jooq.tables.records.SeriesMetadataRecord
 import org.jooq.DSLContext
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 import java.time.LocalDateTime
@ -12,7 +13,8 @@ import java.time.ZoneId

@Component
 class SeriesMetadataDao(
-  private val dsl: DSLContext
+  private val dsl: DSLContext,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : SeriesMetadataRepository {

  private val d = Tables.SERIES_METADATA
@ -117,27 +119,31 @@ class SeriesMetadataDao(

  private fun insertGenres(metadata: SeriesMetadata) {
    if (metadata.genres.isNotEmpty()) {
-      dsl.batch(
-        dsl.insertInto(g, g.SERIES_ID, g.GENRE)
-          .values(null as String?, null)
-      ).also { step ->
-        metadata.genres.forEach {
-          step.bind(metadata.seriesId, it)
-        }
-      }.execute()
+      metadata.genres.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(g, g.SERIES_ID, g.GENRE)
+            .values(null as String?, null)
+        ).also { step ->
+          chunk.forEach {
+            step.bind(metadata.seriesId, it)
+          }
+        }.execute()
+      }
    }
  }

  private fun insertTags(metadata: SeriesMetadata) {
    if (metadata.tags.isNotEmpty()) {
-      dsl.batch(
-        dsl.insertInto(st, st.SERIES_ID, st.TAG)
-          .values(null as String?, null)
-      ).also { step ->
-        metadata.tags.forEach {
-          step.bind(metadata.seriesId, it)
-        }
-      }.execute()
+      metadata.tags.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(st, st.SERIES_ID, st.TAG)
+            .values(null as String?, null)
+        ).also { step ->
+          chunk.forEach {
+            step.bind(metadata.seriesId, it)
+          }
+        }.execute()
+      }
    }
  }

--- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt
+++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt
@ -6,15 +6,16 @@ import org.gotson.komga.domain.persistence.SidecarRepository
 import org.gotson.komga.jooq.Tables
 import org.gotson.komga.jooq.tables.records.SidecarRecord
 import org.jooq.DSLContext
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 import java.net.URL

@Component
 class SidecarDao(
-  private val dsl: DSLContext
+  private val dsl: DSLContext,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : SidecarRepository {
-
  private val sc = Tables.SIDECAR
  private val u = Tables.TEMP_URL_LIST

@ -42,13 +43,15 @@ class SidecarDao(
    dsl.deleteFrom(u).execute()

    if (urls.isNotEmpty()) {
-      dsl.batch(
-        dsl.insertInto(u, u.URL).values(null as String?)
-      ).also { step ->
-        urls.forEach {
-          step.bind(it.toString())
-        }
-      }.execute()
+      urls.chunked(batchSize).forEach { chunk ->
+        dsl.batch(
+          dsl.insertInto(u, u.URL).values(null as String?)
+        ).also { step ->
+          chunk.forEach {
+            step.bind(it.toString())
+          }
+        }.execute()
+      }
    }

    dsl.deleteFrom(sc)