Backup_Repos/komga
1
0
Fork 0
mirror of https://github.com/gotson/komga.git synced 2025-12-20 07:23:34 +01:00
Code Issues Projects Releases Wiki Activity

fix: exceptions SQLITE_TOOBIG could arise when deleting many books or series

Browse source
This commit is contained in:
Gauthier Roebroeck 2021-12-06 11:17:52 +08:00
parent 496695f1e7
commit adf9e14fb2
14 changed files with 100 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
komga/src/flyway/resources/db/migration/sqlite/V20211206100610__temp_table_ids.sql Normal file
View file

@ -0,0 +1,6 @@
CREATE TABLE TEMP_STRING_LIST
(
STRING varchar NOT NULL
);
DROP TABLE TEMP_URL_LIST;

23
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt
View file

@ -29,7 +29,6 @@ class BookDao(
private val m = Tables.MEDIA
private val d = Tables.BOOK_METADATA
private val r = Tables.READ_PROGRESS
private val u = Tables.TEMP_URL_LIST
private val sorts = mapOf(
"createdDate" to b.CREATED_DATE,
@ -67,25 +66,12 @@ class BookDao(
@Transactional
override fun findAllNotDeletedByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection<URL>): Collection<Book> {
// insert urls in a temporary table, else the select size can exceed the statement limit
dsl.deleteFrom(u).execute()
if (urls.isNotEmpty()) {
urls.chunked(batchSize).forEach { chunk ->
dsl.batch(
dsl.insertInto(u, u.URL).values(null as String?)
).also { step ->
chunk.forEach {
step.bind(it.toString())
}
}.execute()
}
}
dsl.insertTempStrings(batchSize, urls.map { it.toString() })
return dsl.selectFrom(b)
.where(b.LIBRARY_ID.eq(libraryId))
.and(b.DELETED_DATE.isNull)
.and(b.URL.notIn(dsl.select(u.URL).from(u)))
.and(b.URL.notIn(dsl.selectTempStrings()))
.fetchInto(b)
.map { it.toDomain() }
}
@ -311,8 +297,11 @@ class BookDao(
dsl.deleteFrom(b).where(b.ID.eq(bookId)).execute()
}
@Transactional
override fun delete(bookIds: Collection<String>) {
dsl.deleteFrom(b).where(b.ID.`in`(bookIds)).execute()
dsl.insertTempStrings(batchSize, bookIds)
dsl.deleteFrom(b).where(b.ID.`in`(dsl.selectTempStrings())).execute()
}
override fun deleteAll() {

8
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt
View file

@ -121,9 +121,11 @@ class BookMetadataAggregationDao(
@Transactional
override fun delete(seriesIds: Collection<String>) {
dsl.deleteFrom(a).where(a.SERIES_ID.`in`(seriesIds)).execute()
dsl.deleteFrom(t).where(t.SERIES_ID.`in`(seriesIds)).execute()
dsl.deleteFrom(d).where(d.SERIES_ID.`in`(seriesIds)).execute()
dsl.insertTempStrings(batchSize, seriesIds)
dsl.deleteFrom(a).where(a.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
dsl.deleteFrom(t).where(t.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
dsl.deleteFrom(d).where(d.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
}
override fun count(): Long = dsl.fetchCount(d).toLong()

8
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt
View file

@ -194,9 +194,11 @@ class BookMetadataDao(
@Transactional
override fun delete(bookIds: Collection<String>) {
dsl.deleteFrom(a).where(a.BOOK_ID.`in`(bookIds)).execute()
dsl.deleteFrom(bt).where(bt.BOOK_ID.`in`(bookIds)).execute()
dsl.deleteFrom(d).where(d.BOOK_ID.`in`(bookIds)).execute()
dsl.insertTempStrings(batchSize, bookIds)
dsl.deleteFrom(a).where(a.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
dsl.deleteFrom(bt).where(bt.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
dsl.deleteFrom(d).where(d.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
}
override fun count(): Long = dsl.fetchCount(d).toLong()

8
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt
View file

@ -183,9 +183,11 @@ class MediaDao(
@Transactional
override fun deleteByBookIds(bookIds: Collection<String>) {
dsl.deleteFrom(p).where(p.BOOK_ID.`in`(bookIds)).execute()
dsl.deleteFrom(f).where(f.BOOK_ID.`in`(bookIds)).execute()
dsl.deleteFrom(m).where(m.BOOK_ID.`in`(bookIds)).execute()
dsl.insertTempStrings(batchSize, bookIds)
dsl.deleteFrom(p).where(p.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
dsl.deleteFrom(f).where(f.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
dsl.deleteFrom(m).where(m.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
}
override fun count(): Long = dsl.fetchCount(m).toLong()

7
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt
View file

@ -10,6 +10,7 @@ import org.gotson.komga.jooq.tables.records.ReadlistRecord
import org.jooq.DSLContext
import org.jooq.Record
import org.jooq.ResultQuery
import org.springframework.beans.factory.annotation.Value
import org.springframework.data.domain.Page
import org.springframework.data.domain.PageImpl
import org.springframework.data.domain.PageRequest
@ -25,6 +26,7 @@ import java.util.SortedMap
class ReadListDao(
private val dsl: DSLContext,
private val luceneHelper: LuceneHelper,
@Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
) : ReadListRepository {
private val rl = Tables.READLIST
@ -212,9 +214,12 @@ class ReadListDao(
.execute()
}
@Transactional
override fun removeBooksFromAll(bookIds: Collection<String>) {
dsl.insertTempStrings(batchSize, bookIds)
dsl.deleteFrom(rlb)
.where(rlb.BOOK_ID.`in`(bookIds))
.where(rlb.BOOK_ID.`in`(dsl.selectTempStrings()))
.execute()
}

24
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt
View file

@ -112,17 +112,24 @@ class ReadProgressDao(
@Transactional
override fun deleteByBookIds(bookIds: Collection<String>) {
dsl.deleteFrom(r).where(r.BOOK_ID.`in`(bookIds)).execute()
dsl.insertTempStrings(batchSize, bookIds)
dsl.deleteFrom(r).where(r.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
aggregateSeriesProgress(bookIds)
}
@Transactional
override fun deleteBySeriesIds(seriesIds: Collection<String>) {
dsl.deleteFrom(rs).where(rs.SERIES_ID.`in`(seriesIds)).execute()
dsl.insertTempStrings(batchSize, seriesIds)
dsl.deleteFrom(rs).where(rs.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
}
@Transactional
override fun deleteByBookIdsAndUserId(bookIds: Collection<String>, userId: String) {
dsl.deleteFrom(r).where(r.BOOK_ID.`in`(bookIds)).and(r.USER_ID.eq(userId)).execute()
dsl.insertTempStrings(batchSize, bookIds)
dsl.deleteFrom(r).where(r.BOOK_ID.`in`(dsl.selectTempStrings())).and(r.USER_ID.eq(userId)).execute()
aggregateSeriesProgress(bookIds, userId)
}
@ -133,13 +140,14 @@ class ReadProgressDao(
}
private fun aggregateSeriesProgress(bookIds: Collection<String>, userId: String? = null) {
val seriesIds = dsl.select(b.SERIES_ID)
dsl.insertTempStrings(batchSize, bookIds)
val seriesIdsQuery = dsl.select(b.SERIES_ID)
.from(b)
.where(b.ID.`in`(bookIds))
.fetch(b.SERIES_ID)
.where(b.ID.`in`(dsl.selectTempStrings()))
dsl.deleteFrom(rs)
.where(rs.SERIES_ID.`in`(seriesIds))
.where(rs.SERIES_ID.`in`(seriesIdsQuery))
.apply { userId?.let { and(rs.USER_ID.eq(it)) } }
.execute()
@ -150,7 +158,7 @@ class ReadProgressDao(
.select(DSL.sum(DSL.`when`(r.COMPLETED.isFalse, 1).otherwise(0)))
.from(b)
.innerJoin(r).on(b.ID.eq(r.BOOK_ID))
.where(b.SERIES_ID.`in`(seriesIds))
.where(b.SERIES_ID.`in`(seriesIdsQuery))
.apply { userId?.let { and(r.USER_ID.eq(it)) } }
.groupBy(b.SERIES_ID, r.USER_ID)
).execute()

6
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt
View file

@ -10,6 +10,7 @@ import org.gotson.komga.jooq.tables.records.CollectionRecord
import org.jooq.DSLContext
import org.jooq.Record
import org.jooq.ResultQuery
import org.springframework.beans.factory.annotation.Value
import org.springframework.data.domain.Page
import org.springframework.data.domain.PageImpl
import org.springframework.data.domain.PageRequest
@ -24,6 +25,7 @@ import java.time.ZoneId
class SeriesCollectionDao(
private val dsl: DSLContext,
private val luceneHelper: LuceneHelper,
@Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
) : SeriesCollectionRepository {
private val c = Tables.COLLECTION
@ -213,8 +215,10 @@ class SeriesCollectionDao(
@Transactional
override fun removeSeriesFromAll(seriesIds: Collection<String>) {
dsl.insertTempStrings(batchSize, seriesIds)
dsl.deleteFrom(cs)
.where(cs.SERIES_ID.`in`(seriesIds))
.where(cs.SERIES_ID.`in`(dsl.selectTempStrings()))
.execute()
}

24
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt
View file

@ -23,7 +23,6 @@ class SeriesDao(
private val s = Tables.SERIES
private val d = Tables.SERIES_METADATA
private val cs = Tables.COLLECTION_SERIES
private val u = Tables.TEMP_URL_LIST
override fun findAll(): Collection<Series> =
dsl.selectFrom(s)
@ -44,26 +43,12 @@ class SeriesDao(
@Transactional
override fun findAllNotDeletedByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection<URL>): List<Series> {
// insert urls in a temporary table, else the select size can exceed the statement limit
dsl.deleteFrom(u).execute()
if (urls.isNotEmpty()) {
urls.chunked(batchSize)
.forEach { chunk ->
dsl.batch(
dsl.insertInto(u, u.URL).values(null as String?)
).also { step ->
chunk.forEach {
step.bind(it.toString())
}
}.execute()
}
}
dsl.insertTempStrings(batchSize, urls.map { it.toString() })
return dsl.selectFrom(s)
.where(s.LIBRARY_ID.eq(libraryId))
.and(s.DELETED_DATE.isNull)
.and(s.URL.notIn(dsl.select(u.URL).from(u)))
.and(s.URL.notIn(dsl.selectTempStrings()))
.fetchInto(s)
.map { it.toDomain() }
}
@ -139,8 +124,11 @@ class SeriesDao(
dsl.deleteFrom(s).execute()
}
@Transactional
override fun delete(seriesIds: Collection<String>) {
dsl.deleteFrom(s).where(s.ID.`in`(seriesIds)).execute()
dsl.insertTempStrings(batchSize, seriesIds)
dsl.deleteFrom(s).where(s.ID.`in`(dsl.selectTempStrings())).execute()
}
override fun count(): Long = dsl.fetchCount(s).toLong()

8
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt
View file

@ -156,9 +156,11 @@ class SeriesMetadataDao(
@Transactional
override fun delete(seriesIds: Collection<String>) {
dsl.deleteFrom(g).where(g.SERIES_ID.`in`(seriesIds)).execute()
dsl.deleteFrom(st).where(st.SERIES_ID.`in`(seriesIds)).execute()
dsl.deleteFrom(d).where(d.SERIES_ID.`in`(seriesIds)).execute()
dsl.insertTempStrings(batchSize, seriesIds)
dsl.deleteFrom(g).where(g.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
dsl.deleteFrom(st).where(st.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
dsl.deleteFrom(d).where(d.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
}
override fun count(): Long = dsl.fetchCount(d).toLong()

18
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt
View file

@ -17,7 +17,6 @@ class SidecarDao(
@Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
) : SidecarRepository {
private val sc = Tables.SIDECAR
private val u = Tables.TEMP_URL_LIST
override fun findAll(): Collection<SidecarStored> =
dsl.selectFrom(sc).fetch().map { it.toDomain() }
@ -39,24 +38,11 @@ class SidecarDao(
@Transactional
override fun deleteByLibraryIdAndUrls(libraryId: String, urls: Collection<URL>) {
// insert urls in a temporary table, else the select size can exceed the statement limit
dsl.deleteFrom(u).execute()
if (urls.isNotEmpty()) {
urls.chunked(batchSize).forEach { chunk ->
dsl.batch(
dsl.insertInto(u, u.URL).values(null as String?)
).also { step ->
chunk.forEach {
step.bind(it.toString())
}
}.execute()
}
}
dsl.insertTempStrings(batchSize, urls.map { it.toString() })
dsl.deleteFrom(sc)
.where(sc.LIBRARY_ID.eq(libraryId))
.and(sc.URL.`in`(dsl.select(u.URL).from(u)))
.and(sc.URL.`in`(dsl.selectTempStrings()))
.execute()
}

9
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailBookDao.kt
View file

@ -5,13 +5,15 @@ import org.gotson.komga.domain.persistence.ThumbnailBookRepository
import org.gotson.komga.jooq.Tables
import org.gotson.komga.jooq.tables.records.ThumbnailBookRecord
import org.jooq.DSLContext
import org.springframework.beans.factory.annotation.Value
import org.springframework.stereotype.Component
import org.springframework.transaction.annotation.Transactional
import java.net.URL
@Component
class ThumbnailBookDao(
private val dsl: DSLContext
private val dsl: DSLContext,
@Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
) : ThumbnailBookRepository {
private val tb = Tables.THUMBNAIL_BOOK
@ -82,8 +84,11 @@ class ThumbnailBookDao(
dsl.deleteFrom(tb).where(tb.BOOK_ID.eq(bookId)).execute()
}
@Transactional
override fun deleteByBookIds(bookIds: Collection<String>) {
dsl.deleteFrom(tb).where(tb.BOOK_ID.`in`(bookIds)).execute()
dsl.insertTempStrings(batchSize, bookIds)
dsl.deleteFrom(tb).where(tb.BOOK_ID.`in`(dsl.selectTempStrings())).execute()
}
override fun deleteByBookIdAndType(bookId: String, type: ThumbnailBook.Type) {

9
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailSeriesDao.kt
View file

@ -5,13 +5,15 @@ import org.gotson.komga.domain.persistence.ThumbnailSeriesRepository
import org.gotson.komga.jooq.Tables
import org.gotson.komga.jooq.tables.records.ThumbnailSeriesRecord
import org.jooq.DSLContext
import org.springframework.beans.factory.annotation.Value
import org.springframework.stereotype.Component
import org.springframework.transaction.annotation.Transactional
import java.net.URL
@Component
class ThumbnailSeriesDao(
private val dsl: DSLContext
private val dsl: DSLContext,
@Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int,
) : ThumbnailSeriesRepository {
private val ts = Tables.THUMBNAIL_SERIES
@ -70,8 +72,11 @@ class ThumbnailSeriesDao(
dsl.deleteFrom(ts).where(ts.SERIES_ID.eq(seriesId)).execute()
}
@Transactional
override fun deleteBySeriesIds(seriesIds: Collection<String>) {
dsl.deleteFrom(ts).where(ts.SERIES_ID.`in`(seriesIds)).execute()
dsl.insertTempStrings(batchSize, seriesIds)
dsl.deleteFrom(ts).where(ts.SERIES_ID.`in`(dsl.selectTempStrings())).execute()
}
private fun ThumbnailSeriesRecord.toDomain() =

19
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt
View file

@ -1,7 +1,9 @@
package org.gotson.komga.infrastructure.jooq
import org.gotson.komga.infrastructure.datasource.SqliteUdfDataSource
import org.gotson.komga.jooq.Tables
import org.jooq.Condition
import org.jooq.DSLContext
import org.jooq.Field
import org.jooq.SortField
import org.jooq.impl.DSL
@ -44,3 +46,20 @@ fun LocalDateTime.toCurrentTimeZone(): LocalDateTime =
fun Field<String>.udfStripAccents() =
DSL.function(SqliteUdfDataSource.udfStripAccents, String::class.java, this)
fun DSLContext.insertTempStrings(batchSize: Int, collection: Collection<String>) {
this.deleteFrom(Tables.TEMP_STRING_LIST).execute()
if (collection.isNotEmpty()) {
collection.chunked(batchSize).forEach { chunk ->
this.batch(
this.insertInto(Tables.TEMP_STRING_LIST, Tables.TEMP_STRING_LIST.STRING).values(null as String?)
).also { step ->
chunk.forEach {
step.bind(it)
}
}.execute()
}
}
}
fun DSLContext.selectTempStrings() = this.select(Tables.TEMP_STRING_LIST.STRING).from(Tables.TEMP_STRING_LIST)