fix: exceptions SQLITE_TOOBIG could arise when deleting many books or series

komga/src/flyway/resources/db/migration/sqlite/V20211206100610__temp_table_ids.sql

@@ -0,0 +1,6 @@
+CREATE TABLE TEMP_STRING_LIST
+(
+    STRING varchar NOT NULL
+);
+
+DROP TABLE TEMP_URL_LIST;

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt

@@ -29,7 +29,6 @@ class BookDao(
   private val m = Tables.MEDIA
   private val d = Tables.BOOK_METADATA
   private val r = Tables.READ_PROGRESS
-  private val u = Tables.TEMP_URL_LIST
 
   private val sorts = mapOf(
     "createdDate" to b.CREATED_DATE,
@@ -67,25 +66,12 @@ class BookDao(
 
   @Transactional
   override fun findAllNotDeletedByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection<URL>): Collection<Book> {
-    // insert urls in a temporary table, else the select size can exceed the statement limit
+    dsl.insertTempStrings(batchSize, urls.map { it.toString() })
-    dsl.deleteFrom(u).execute()
-
-    if (urls.isNotEmpty()) {
-      urls.chunked(batchSize).forEach { chunk ->
-        dsl.batch(
-          dsl.insertInto(u, u.URL).values(null as String?)
-        ).also { step ->
-          chunk.forEach {
-            step.bind(it.toString())
-          }
-        }.execute()
-      }
-    }
 
     return dsl.selectFrom(b)
       .where(b.LIBRARY_ID.eq(libraryId))
       .and(b.DELETED_DATE.isNull)
-      .and(b.URL.notIn(dsl.select(u.URL).from(u)))
+      .and(b.URL.notIn(dsl.selectTempStrings()))
       .fetchInto(b)
       .map { it.toDomain() }
   }
@@ -311,8 +297,11 @@ class BookDao(
     dsl.deleteFrom(b).where(b.ID.eq(bookId)).execute()
   }
 
+  @Transactional
   override fun delete(bookIds: Collection<String>) {
-    dsl.deleteFrom(b).where(b.ID.`in`(bookIds)).execute()
+    dsl.insertTempStrings(batchSize, bookIds)
+
+    dsl.deleteFrom(b).where(b.ID.`in`(dsl.selectTempStrings())).execute()
   }
 
   override fun deleteAll() {

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt

@@ -121,9 +121,11 @@ class BookMetadataAggregationDao(
 
   @Transactional
   override fun delete(seriesIds: Collection<String>) {
-    dsl.deleteFrom(a).where(a.SERIES_ID.`in`(seriesIds)).execute()
+    dsl.insertTempStrings(batchSize, seriesIds)
-    dsl.deleteFrom(t).where(t.SERIES_ID.`in`(seriesIds)).execute()
+
-    dsl.deleteFrom(d).where(d.SERIES_ID.`in`(seriesIds)).execute()
+    dsl.deleteFrom(a).where(a.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
+    dsl.deleteFrom(t).where(t.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
+    dsl.deleteFrom(d).where(d.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
   }
 
   override fun count(): Long = dsl.fetchCount(d).toLong()

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt

@@ -194,9 +194,11 @@ class BookMetadataDao(
 
   @Transactional
   override fun delete(bookIds: Collection<String>) {
-    dsl.deleteFrom(a).where(a.BOOK_ID.`in`(bookIds)).execute()
+    dsl.insertTempStrings(batchSize, bookIds)
-    dsl.deleteFrom(bt).where(bt.BOOK_ID.`in`(bookIds)).execute()
+
-    dsl.deleteFrom(d).where(d.BOOK_ID.`in`(bookIds)).execute()
+    dsl.deleteFrom(a).where(a.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
+    dsl.deleteFrom(bt).where(bt.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
+    dsl.deleteFrom(d).where(d.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
   }
 
   override fun count(): Long = dsl.fetchCount(d).toLong()

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt

@@ -183,9 +183,11 @@ class MediaDao(
 
   @Transactional
   override fun deleteByBookIds(bookIds: Collection<String>) {
-    dsl.deleteFrom(p).where(p.BOOK_ID.`in`(bookIds)).execute()
+    dsl.insertTempStrings(batchSize, bookIds)
-    dsl.deleteFrom(f).where(f.BOOK_ID.`in`(bookIds)).execute()
+
-    dsl.deleteFrom(m).where(m.BOOK_ID.`in`(bookIds)).execute()
+    dsl.deleteFrom(p).where(p.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
+    dsl.deleteFrom(f).where(f.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
+    dsl.deleteFrom(m).where(m.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
   }
 
   override fun count(): Long = dsl.fetchCount(m).toLong()

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt

@@ -10,6 +10,7 @@ import org.gotson.komga.jooq.tables.records.ReadlistRecord
 import org.jooq.DSLContext
 import org.jooq.Record
 import org.jooq.ResultQuery
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.data.domain.Page
 import org.springframework.data.domain.PageImpl
 import org.springframework.data.domain.PageRequest
@@ -25,6 +26,7 @@ import java.util.SortedMap
 class ReadListDao(
   private val dsl: DSLContext,
   private val luceneHelper: LuceneHelper,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : ReadListRepository {
 
   private val rl = Tables.READLIST
@@ -212,9 +214,12 @@ class ReadListDao(
       .execute()
   }
 
+  @Transactional
   override fun removeBooksFromAll(bookIds: Collection<String>) {
+    dsl.insertTempStrings(batchSize, bookIds)
+
     dsl.deleteFrom(rlb)
-      .where(rlb.BOOK_ID.`in`(bookIds))
+      .where(rlb.BOOK_ID.`in`(dsl.selectTempStrings()))
       .execute()
   }
 

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt

@@ -112,17 +112,24 @@ class ReadProgressDao(
 
   @Transactional
   override fun deleteByBookIds(bookIds: Collection<String>) {
-    dsl.deleteFrom(r).where(r.BOOK_ID.`in`(bookIds)).execute()
+    dsl.insertTempStrings(batchSize, bookIds)
+
+    dsl.deleteFrom(r).where(r.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
     aggregateSeriesProgress(bookIds)
   }
 
+  @Transactional
   override fun deleteBySeriesIds(seriesIds: Collection<String>) {
-    dsl.deleteFrom(rs).where(rs.SERIES_ID.`in`(seriesIds)).execute()
+    dsl.insertTempStrings(batchSize, seriesIds)
+
+    dsl.deleteFrom(rs).where(rs.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
   }
 
   @Transactional
   override fun deleteByBookIdsAndUserId(bookIds: Collection<String>, userId: String) {
-    dsl.deleteFrom(r).where(r.BOOK_ID.`in`(bookIds)).and(r.USER_ID.eq(userId)).execute()
+    dsl.insertTempStrings(batchSize, bookIds)
+
+    dsl.deleteFrom(r).where(r.BOOK_ID.`in`(dsl.selectTempStrings())).and(r.USER_ID.eq(userId)).execute()
     aggregateSeriesProgress(bookIds, userId)
   }
 
@@ -133,13 +140,14 @@ class ReadProgressDao(
   }
 
   private fun aggregateSeriesProgress(bookIds: Collection<String>, userId: String? = null) {
-    val seriesIds = dsl.select(b.SERIES_ID)
+    dsl.insertTempStrings(batchSize, bookIds)
+
+    val seriesIdsQuery = dsl.select(b.SERIES_ID)
       .from(b)
-      .where(b.ID.`in`(bookIds))
+      .where(b.ID.`in`(dsl.selectTempStrings()))
-      .fetch(b.SERIES_ID)
 
     dsl.deleteFrom(rs)
-      .where(rs.SERIES_ID.`in`(seriesIds))
+      .where(rs.SERIES_ID.`in`(seriesIdsQuery))
       .apply { userId?.let { and(rs.USER_ID.eq(it)) } }
       .execute()
 
@@ -150,7 +158,7 @@ class ReadProgressDao(
           .select(DSL.sum(DSL.`when`(r.COMPLETED.isFalse, 1).otherwise(0)))
           .from(b)
           .innerJoin(r).on(b.ID.eq(r.BOOK_ID))
-          .where(b.SERIES_ID.`in`(seriesIds))
+          .where(b.SERIES_ID.`in`(seriesIdsQuery))
           .apply { userId?.let { and(r.USER_ID.eq(it)) } }
           .groupBy(b.SERIES_ID, r.USER_ID)
       ).execute()

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt

@@ -10,6 +10,7 @@ import org.gotson.komga.jooq.tables.records.CollectionRecord
 import org.jooq.DSLContext
 import org.jooq.Record
 import org.jooq.ResultQuery
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.data.domain.Page
 import org.springframework.data.domain.PageImpl
 import org.springframework.data.domain.PageRequest
@@ -24,6 +25,7 @@ import java.time.ZoneId
 class SeriesCollectionDao(
   private val dsl: DSLContext,
   private val luceneHelper: LuceneHelper,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : SeriesCollectionRepository {
 
   private val c = Tables.COLLECTION
@@ -213,8 +215,10 @@ class SeriesCollectionDao(
 
   @Transactional
   override fun removeSeriesFromAll(seriesIds: Collection<String>) {
+    dsl.insertTempStrings(batchSize, seriesIds)
+
     dsl.deleteFrom(cs)
-      .where(cs.SERIES_ID.`in`(seriesIds))
+      .where(cs.SERIES_ID.`in`(dsl.selectTempStrings()))
       .execute()
   }
 

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt

@@ -23,7 +23,6 @@ class SeriesDao(
   private val s = Tables.SERIES
   private val d = Tables.SERIES_METADATA
   private val cs = Tables.COLLECTION_SERIES
-  private val u = Tables.TEMP_URL_LIST
 
   override fun findAll(): Collection<Series> =
     dsl.selectFrom(s)
@@ -44,26 +43,12 @@ class SeriesDao(
 
   @Transactional
   override fun findAllNotDeletedByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection<URL>): List<Series> {
-    // insert urls in a temporary table, else the select size can exceed the statement limit
+    dsl.insertTempStrings(batchSize, urls.map { it.toString() })
-    dsl.deleteFrom(u).execute()
-
-    if (urls.isNotEmpty()) {
-      urls.chunked(batchSize)
-        .forEach { chunk ->
-          dsl.batch(
-            dsl.insertInto(u, u.URL).values(null as String?)
-          ).also { step ->
-            chunk.forEach {
-              step.bind(it.toString())
-            }
-          }.execute()
-        }
-    }
 
     return dsl.selectFrom(s)
       .where(s.LIBRARY_ID.eq(libraryId))
       .and(s.DELETED_DATE.isNull)
-      .and(s.URL.notIn(dsl.select(u.URL).from(u)))
+      .and(s.URL.notIn(dsl.selectTempStrings()))
       .fetchInto(s)
       .map { it.toDomain() }
   }
@@ -139,8 +124,11 @@ class SeriesDao(
     dsl.deleteFrom(s).execute()
   }
 
+  @Transactional
   override fun delete(seriesIds: Collection<String>) {
-    dsl.deleteFrom(s).where(s.ID.`in`(seriesIds)).execute()
+    dsl.insertTempStrings(batchSize, seriesIds)
+
+    dsl.deleteFrom(s).where(s.ID.`in`(dsl.selectTempStrings())).execute()
   }
 
   override fun count(): Long = dsl.fetchCount(s).toLong()

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt

@@ -156,9 +156,11 @@ class SeriesMetadataDao(
 
   @Transactional
   override fun delete(seriesIds: Collection<String>) {
-    dsl.deleteFrom(g).where(g.SERIES_ID.`in`(seriesIds)).execute()
+    dsl.insertTempStrings(batchSize, seriesIds)
-    dsl.deleteFrom(st).where(st.SERIES_ID.`in`(seriesIds)).execute()
+
-    dsl.deleteFrom(d).where(d.SERIES_ID.`in`(seriesIds)).execute()
+    dsl.deleteFrom(g).where(g.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
+    dsl.deleteFrom(st).where(st.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
+    dsl.deleteFrom(d).where(d.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
   }
 
   override fun count(): Long = dsl.fetchCount(d).toLong()

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt

@@ -17,7 +17,6 @@ class SidecarDao(
   @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : SidecarRepository {
   private val sc = Tables.SIDECAR
-  private val u = Tables.TEMP_URL_LIST
 
   override fun findAll(): Collection<SidecarStored> =
     dsl.selectFrom(sc).fetch().map { it.toDomain() }
@@ -39,24 +38,11 @@ class SidecarDao(
 
   @Transactional
   override fun deleteByLibraryIdAndUrls(libraryId: String, urls: Collection<URL>) {
-    // insert urls in a temporary table, else the select size can exceed the statement limit
+    dsl.insertTempStrings(batchSize, urls.map { it.toString() })
-    dsl.deleteFrom(u).execute()
-
-    if (urls.isNotEmpty()) {
-      urls.chunked(batchSize).forEach { chunk ->
-        dsl.batch(
-          dsl.insertInto(u, u.URL).values(null as String?)
-        ).also { step ->
-          chunk.forEach {
-            step.bind(it.toString())
-          }
-        }.execute()
-      }
-    }
 
     dsl.deleteFrom(sc)
       .where(sc.LIBRARY_ID.eq(libraryId))
-      .and(sc.URL.`in`(dsl.select(u.URL).from(u)))
+      .and(sc.URL.`in`(dsl.selectTempStrings()))
       .execute()
   }
 

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailBookDao.kt

@@ -5,13 +5,15 @@ import org.gotson.komga.domain.persistence.ThumbnailBookRepository
 import org.gotson.komga.jooq.Tables
 import org.gotson.komga.jooq.tables.records.ThumbnailBookRecord
 import org.jooq.DSLContext
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 import java.net.URL
 
 @Component
 class ThumbnailBookDao(
-  private val dsl: DSLContext
+  private val dsl: DSLContext,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : ThumbnailBookRepository {
   private val tb = Tables.THUMBNAIL_BOOK
 
@@ -82,8 +84,11 @@ class ThumbnailBookDao(
     dsl.deleteFrom(tb).where(tb.BOOK_ID.eq(bookId)).execute()
   }
 
+  @Transactional
   override fun deleteByBookIds(bookIds: Collection<String>) {
-    dsl.deleteFrom(tb).where(tb.BOOK_ID.`in`(bookIds)).execute()
+    dsl.insertTempStrings(batchSize, bookIds)
+
+    dsl.deleteFrom(tb).where(tb.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
   }
 
   override fun deleteByBookIdAndType(bookId: String, type: ThumbnailBook.Type) {

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailSeriesDao.kt

@@ -5,13 +5,15 @@ import org.gotson.komga.domain.persistence.ThumbnailSeriesRepository
 import org.gotson.komga.jooq.Tables
 import org.gotson.komga.jooq.tables.records.ThumbnailSeriesRecord
 import org.jooq.DSLContext
+import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Component
 import org.springframework.transaction.annotation.Transactional
 import java.net.URL
 
 @Component
 class ThumbnailSeriesDao(
-  private val dsl: DSLContext
+  private val dsl: DSLContext,
+  @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
 ) : ThumbnailSeriesRepository {
   private val ts = Tables.THUMBNAIL_SERIES
 
@@ -70,8 +72,11 @@ class ThumbnailSeriesDao(
     dsl.deleteFrom(ts).where(ts.SERIES_ID.eq(seriesId)).execute()
   }
 
+  @Transactional
   override fun deleteBySeriesIds(seriesIds: Collection<String>) {
-    dsl.deleteFrom(ts).where(ts.SERIES_ID.`in`(seriesIds)).execute()
+    dsl.insertTempStrings(batchSize, seriesIds)
+
+    dsl.deleteFrom(ts).where(ts.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
   }
 
   private fun ThumbnailSeriesRecord.toDomain() =

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt

@@ -1,7 +1,9 @@
 package org.gotson.komga.infrastructure.jooq
 
 import org.gotson.komga.infrastructure.datasource.SqliteUdfDataSource
+import org.gotson.komga.jooq.Tables
 import org.jooq.Condition
+import org.jooq.DSLContext
 import org.jooq.Field
 import org.jooq.SortField
 import org.jooq.impl.DSL
@@ -44,3 +46,20 @@ fun LocalDateTime.toCurrentTimeZone(): LocalDateTime =
 
 fun Field<String>.udfStripAccents() =
   DSL.function(SqliteUdfDataSource.udfStripAccents, String::class.java, this)
+
+fun DSLContext.insertTempStrings(batchSize: Int, collection: Collection<String>) {
+  this.deleteFrom(Tables.TEMP_STRING_LIST).execute()
+  if (collection.isNotEmpty()) {
+    collection.chunked(batchSize).forEach { chunk ->
+      this.batch(
+        this.insertInto(Tables.TEMP_STRING_LIST, Tables.TEMP_STRING_LIST.STRING).values(null as String?)
+      ).also { step ->
+        chunk.forEach {
+          step.bind(it)
+        }
+      }.execute()
+    }
+  }
+}
+
+fun DSLContext.selectTempStrings() = this.select(Tables.TEMP_STRING_LIST.STRING).from(Tables.TEMP_STRING_LIST)