diff --git a/komga/src/flyway/resources/db/migration/sqlite/V20211206100610__temp_table_ids.sql b/komga/src/flyway/resources/db/migration/sqlite/V20211206100610__temp_table_ids.sql new file mode 100644 index 000000000..7f6d7b099 --- /dev/null +++ b/komga/src/flyway/resources/db/migration/sqlite/V20211206100610__temp_table_ids.sql @@ -0,0 +1,6 @@ +CREATE TABLE TEMP_STRING_LIST +( + STRING varchar NOT NULL +); + +DROP TABLE TEMP_URL_LIST; diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt index edf125a1c..ffadfe0ce 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt @@ -29,7 +29,6 @@ class BookDao( private val m = Tables.MEDIA private val d = Tables.BOOK_METADATA private val r = Tables.READ_PROGRESS - private val u = Tables.TEMP_URL_LIST private val sorts = mapOf( "createdDate" to b.CREATED_DATE, @@ -67,25 +66,12 @@ class BookDao( @Transactional override fun findAllNotDeletedByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection): Collection { - // insert urls in a temporary table, else the select size can exceed the statement limit - dsl.deleteFrom(u).execute() - - if (urls.isNotEmpty()) { - urls.chunked(batchSize).forEach { chunk -> - dsl.batch( - dsl.insertInto(u, u.URL).values(null as String?) - ).also { step -> - chunk.forEach { - step.bind(it.toString()) - } - }.execute() - } - } + dsl.insertTempStrings(batchSize, urls.map { it.toString() }) return dsl.selectFrom(b) .where(b.LIBRARY_ID.eq(libraryId)) .and(b.DELETED_DATE.isNull) - .and(b.URL.notIn(dsl.select(u.URL).from(u))) + .and(b.URL.notIn(dsl.selectTempStrings())) .fetchInto(b) .map { it.toDomain() } } @@ -311,8 +297,11 @@ class BookDao( dsl.deleteFrom(b).where(b.ID.eq(bookId)).execute() } + @Transactional override fun delete(bookIds: Collection) { - dsl.deleteFrom(b).where(b.ID.`in`(bookIds)).execute() + dsl.insertTempStrings(batchSize, bookIds) + + dsl.deleteFrom(b).where(b.ID.`in`(dsl.selectTempStrings())).execute() } override fun deleteAll() { diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt index 16624e0ce..4b3cdcf43 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt @@ -121,9 +121,11 @@ class BookMetadataAggregationDao( @Transactional override fun delete(seriesIds: Collection) { - dsl.deleteFrom(a).where(a.SERIES_ID.`in`(seriesIds)).execute() - dsl.deleteFrom(t).where(t.SERIES_ID.`in`(seriesIds)).execute() - dsl.deleteFrom(d).where(d.SERIES_ID.`in`(seriesIds)).execute() + dsl.insertTempStrings(batchSize, seriesIds) + + dsl.deleteFrom(a).where(a.SERIES_ID.`in`(dsl.selectTempStrings())).execute() + dsl.deleteFrom(t).where(t.SERIES_ID.`in`(dsl.selectTempStrings())).execute() + dsl.deleteFrom(d).where(d.SERIES_ID.`in`(dsl.selectTempStrings())).execute() } override fun count(): Long = dsl.fetchCount(d).toLong() diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt index c8f5825ef..8f73ed8dc 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataDao.kt @@ -194,9 +194,11 @@ class BookMetadataDao( @Transactional override fun delete(bookIds: Collection) { - dsl.deleteFrom(a).where(a.BOOK_ID.`in`(bookIds)).execute() - dsl.deleteFrom(bt).where(bt.BOOK_ID.`in`(bookIds)).execute() - dsl.deleteFrom(d).where(d.BOOK_ID.`in`(bookIds)).execute() + dsl.insertTempStrings(batchSize, bookIds) + + dsl.deleteFrom(a).where(a.BOOK_ID.`in`(dsl.selectTempStrings())).execute() + dsl.deleteFrom(bt).where(bt.BOOK_ID.`in`(dsl.selectTempStrings())).execute() + dsl.deleteFrom(d).where(d.BOOK_ID.`in`(dsl.selectTempStrings())).execute() } override fun count(): Long = dsl.fetchCount(d).toLong() diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt index bf685b0cb..ee5b49dd5 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt @@ -183,9 +183,11 @@ class MediaDao( @Transactional override fun deleteByBookIds(bookIds: Collection) { - dsl.deleteFrom(p).where(p.BOOK_ID.`in`(bookIds)).execute() - dsl.deleteFrom(f).where(f.BOOK_ID.`in`(bookIds)).execute() - dsl.deleteFrom(m).where(m.BOOK_ID.`in`(bookIds)).execute() + dsl.insertTempStrings(batchSize, bookIds) + + dsl.deleteFrom(p).where(p.BOOK_ID.`in`(dsl.selectTempStrings())).execute() + dsl.deleteFrom(f).where(f.BOOK_ID.`in`(dsl.selectTempStrings())).execute() + dsl.deleteFrom(m).where(m.BOOK_ID.`in`(dsl.selectTempStrings())).execute() } override fun count(): Long = dsl.fetchCount(m).toLong() diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt index 9f6d14496..580be133a 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt @@ -10,6 +10,7 @@ import org.gotson.komga.jooq.tables.records.ReadlistRecord import org.jooq.DSLContext import org.jooq.Record import org.jooq.ResultQuery +import org.springframework.beans.factory.annotation.Value import org.springframework.data.domain.Page import org.springframework.data.domain.PageImpl import org.springframework.data.domain.PageRequest @@ -25,6 +26,7 @@ import java.util.SortedMap class ReadListDao( private val dsl: DSLContext, private val luceneHelper: LuceneHelper, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : ReadListRepository { private val rl = Tables.READLIST @@ -212,9 +214,12 @@ class ReadListDao( .execute() } + @Transactional override fun removeBooksFromAll(bookIds: Collection) { + dsl.insertTempStrings(batchSize, bookIds) + dsl.deleteFrom(rlb) - .where(rlb.BOOK_ID.`in`(bookIds)) + .where(rlb.BOOK_ID.`in`(dsl.selectTempStrings())) .execute() } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt index 2464031da..eb0185a17 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadProgressDao.kt @@ -112,17 +112,24 @@ class ReadProgressDao( @Transactional override fun deleteByBookIds(bookIds: Collection) { - dsl.deleteFrom(r).where(r.BOOK_ID.`in`(bookIds)).execute() + dsl.insertTempStrings(batchSize, bookIds) + + dsl.deleteFrom(r).where(r.BOOK_ID.`in`(dsl.selectTempStrings())).execute() aggregateSeriesProgress(bookIds) } + @Transactional override fun deleteBySeriesIds(seriesIds: Collection) { - dsl.deleteFrom(rs).where(rs.SERIES_ID.`in`(seriesIds)).execute() + dsl.insertTempStrings(batchSize, seriesIds) + + dsl.deleteFrom(rs).where(rs.SERIES_ID.`in`(dsl.selectTempStrings())).execute() } @Transactional override fun deleteByBookIdsAndUserId(bookIds: Collection, userId: String) { - dsl.deleteFrom(r).where(r.BOOK_ID.`in`(bookIds)).and(r.USER_ID.eq(userId)).execute() + dsl.insertTempStrings(batchSize, bookIds) + + dsl.deleteFrom(r).where(r.BOOK_ID.`in`(dsl.selectTempStrings())).and(r.USER_ID.eq(userId)).execute() aggregateSeriesProgress(bookIds, userId) } @@ -133,13 +140,14 @@ class ReadProgressDao( } private fun aggregateSeriesProgress(bookIds: Collection, userId: String? = null) { - val seriesIds = dsl.select(b.SERIES_ID) + dsl.insertTempStrings(batchSize, bookIds) + + val seriesIdsQuery = dsl.select(b.SERIES_ID) .from(b) - .where(b.ID.`in`(bookIds)) - .fetch(b.SERIES_ID) + .where(b.ID.`in`(dsl.selectTempStrings())) dsl.deleteFrom(rs) - .where(rs.SERIES_ID.`in`(seriesIds)) + .where(rs.SERIES_ID.`in`(seriesIdsQuery)) .apply { userId?.let { and(rs.USER_ID.eq(it)) } } .execute() @@ -150,7 +158,7 @@ class ReadProgressDao( .select(DSL.sum(DSL.`when`(r.COMPLETED.isFalse, 1).otherwise(0))) .from(b) .innerJoin(r).on(b.ID.eq(r.BOOK_ID)) - .where(b.SERIES_ID.`in`(seriesIds)) + .where(b.SERIES_ID.`in`(seriesIdsQuery)) .apply { userId?.let { and(r.USER_ID.eq(it)) } } .groupBy(b.SERIES_ID, r.USER_ID) ).execute() diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt index 88b7724d3..d06a2b72f 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt @@ -10,6 +10,7 @@ import org.gotson.komga.jooq.tables.records.CollectionRecord import org.jooq.DSLContext import org.jooq.Record import org.jooq.ResultQuery +import org.springframework.beans.factory.annotation.Value import org.springframework.data.domain.Page import org.springframework.data.domain.PageImpl import org.springframework.data.domain.PageRequest @@ -24,6 +25,7 @@ import java.time.ZoneId class SeriesCollectionDao( private val dsl: DSLContext, private val luceneHelper: LuceneHelper, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : SeriesCollectionRepository { private val c = Tables.COLLECTION @@ -213,8 +215,10 @@ class SeriesCollectionDao( @Transactional override fun removeSeriesFromAll(seriesIds: Collection) { + dsl.insertTempStrings(batchSize, seriesIds) + dsl.deleteFrom(cs) - .where(cs.SERIES_ID.`in`(seriesIds)) + .where(cs.SERIES_ID.`in`(dsl.selectTempStrings())) .execute() } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt index 0e0dd08b1..ce4b38c81 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt @@ -23,7 +23,6 @@ class SeriesDao( private val s = Tables.SERIES private val d = Tables.SERIES_METADATA private val cs = Tables.COLLECTION_SERIES - private val u = Tables.TEMP_URL_LIST override fun findAll(): Collection = dsl.selectFrom(s) @@ -44,26 +43,12 @@ class SeriesDao( @Transactional override fun findAllNotDeletedByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection): List { - // insert urls in a temporary table, else the select size can exceed the statement limit - dsl.deleteFrom(u).execute() - - if (urls.isNotEmpty()) { - urls.chunked(batchSize) - .forEach { chunk -> - dsl.batch( - dsl.insertInto(u, u.URL).values(null as String?) - ).also { step -> - chunk.forEach { - step.bind(it.toString()) - } - }.execute() - } - } + dsl.insertTempStrings(batchSize, urls.map { it.toString() }) return dsl.selectFrom(s) .where(s.LIBRARY_ID.eq(libraryId)) .and(s.DELETED_DATE.isNull) - .and(s.URL.notIn(dsl.select(u.URL).from(u))) + .and(s.URL.notIn(dsl.selectTempStrings())) .fetchInto(s) .map { it.toDomain() } } @@ -139,8 +124,11 @@ class SeriesDao( dsl.deleteFrom(s).execute() } + @Transactional override fun delete(seriesIds: Collection) { - dsl.deleteFrom(s).where(s.ID.`in`(seriesIds)).execute() + dsl.insertTempStrings(batchSize, seriesIds) + + dsl.deleteFrom(s).where(s.ID.`in`(dsl.selectTempStrings())).execute() } override fun count(): Long = dsl.fetchCount(s).toLong() diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt index f1683744d..0fa81d1e7 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesMetadataDao.kt @@ -156,9 +156,11 @@ class SeriesMetadataDao( @Transactional override fun delete(seriesIds: Collection) { - dsl.deleteFrom(g).where(g.SERIES_ID.`in`(seriesIds)).execute() - dsl.deleteFrom(st).where(st.SERIES_ID.`in`(seriesIds)).execute() - dsl.deleteFrom(d).where(d.SERIES_ID.`in`(seriesIds)).execute() + dsl.insertTempStrings(batchSize, seriesIds) + + dsl.deleteFrom(g).where(g.SERIES_ID.`in`(dsl.selectTempStrings())).execute() + dsl.deleteFrom(st).where(st.SERIES_ID.`in`(dsl.selectTempStrings())).execute() + dsl.deleteFrom(d).where(d.SERIES_ID.`in`(dsl.selectTempStrings())).execute() } override fun count(): Long = dsl.fetchCount(d).toLong() diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt index c170666c9..bdcefdbf5 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt @@ -17,7 +17,6 @@ class SidecarDao( @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : SidecarRepository { private val sc = Tables.SIDECAR - private val u = Tables.TEMP_URL_LIST override fun findAll(): Collection = dsl.selectFrom(sc).fetch().map { it.toDomain() } @@ -39,24 +38,11 @@ class SidecarDao( @Transactional override fun deleteByLibraryIdAndUrls(libraryId: String, urls: Collection) { - // insert urls in a temporary table, else the select size can exceed the statement limit - dsl.deleteFrom(u).execute() - - if (urls.isNotEmpty()) { - urls.chunked(batchSize).forEach { chunk -> - dsl.batch( - dsl.insertInto(u, u.URL).values(null as String?) - ).also { step -> - chunk.forEach { - step.bind(it.toString()) - } - }.execute() - } - } + dsl.insertTempStrings(batchSize, urls.map { it.toString() }) dsl.deleteFrom(sc) .where(sc.LIBRARY_ID.eq(libraryId)) - .and(sc.URL.`in`(dsl.select(u.URL).from(u))) + .and(sc.URL.`in`(dsl.selectTempStrings())) .execute() } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailBookDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailBookDao.kt index e91513aea..06b580f01 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailBookDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailBookDao.kt @@ -5,13 +5,15 @@ import org.gotson.komga.domain.persistence.ThumbnailBookRepository import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.ThumbnailBookRecord import org.jooq.DSLContext +import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.net.URL @Component class ThumbnailBookDao( - private val dsl: DSLContext + private val dsl: DSLContext, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : ThumbnailBookRepository { private val tb = Tables.THUMBNAIL_BOOK @@ -82,8 +84,11 @@ class ThumbnailBookDao( dsl.deleteFrom(tb).where(tb.BOOK_ID.eq(bookId)).execute() } + @Transactional override fun deleteByBookIds(bookIds: Collection) { - dsl.deleteFrom(tb).where(tb.BOOK_ID.`in`(bookIds)).execute() + dsl.insertTempStrings(batchSize, bookIds) + + dsl.deleteFrom(tb).where(tb.BOOK_ID.`in`(dsl.selectTempStrings())).execute() } override fun deleteByBookIdAndType(bookId: String, type: ThumbnailBook.Type) { diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailSeriesDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailSeriesDao.kt index 4efa34e65..fbbcb07db 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailSeriesDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ThumbnailSeriesDao.kt @@ -5,13 +5,15 @@ import org.gotson.komga.domain.persistence.ThumbnailSeriesRepository import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.ThumbnailSeriesRecord import org.jooq.DSLContext +import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.net.URL @Component class ThumbnailSeriesDao( - private val dsl: DSLContext + private val dsl: DSLContext, + @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : ThumbnailSeriesRepository { private val ts = Tables.THUMBNAIL_SERIES @@ -70,8 +72,11 @@ class ThumbnailSeriesDao( dsl.deleteFrom(ts).where(ts.SERIES_ID.eq(seriesId)).execute() } + @Transactional override fun deleteBySeriesIds(seriesIds: Collection) { - dsl.deleteFrom(ts).where(ts.SERIES_ID.`in`(seriesIds)).execute() + dsl.insertTempStrings(batchSize, seriesIds) + + dsl.deleteFrom(ts).where(ts.SERIES_ID.`in`(dsl.selectTempStrings())).execute() } private fun ThumbnailSeriesRecord.toDomain() = diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt index b88850117..71019d6de 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt @@ -1,7 +1,9 @@ package org.gotson.komga.infrastructure.jooq import org.gotson.komga.infrastructure.datasource.SqliteUdfDataSource +import org.gotson.komga.jooq.Tables import org.jooq.Condition +import org.jooq.DSLContext import org.jooq.Field import org.jooq.SortField import org.jooq.impl.DSL @@ -44,3 +46,20 @@ fun LocalDateTime.toCurrentTimeZone(): LocalDateTime = fun Field.udfStripAccents() = DSL.function(SqliteUdfDataSource.udfStripAccents, String::class.java, this) + +fun DSLContext.insertTempStrings(batchSize: Int, collection: Collection) { + this.deleteFrom(Tables.TEMP_STRING_LIST).execute() + if (collection.isNotEmpty()) { + collection.chunked(batchSize).forEach { chunk -> + this.batch( + this.insertInto(Tables.TEMP_STRING_LIST, Tables.TEMP_STRING_LIST.STRING).values(null as String?) + ).also { step -> + chunk.forEach { + step.bind(it) + } + }.execute() + } + } +} + +fun DSLContext.selectTempStrings() = this.select(Tables.TEMP_STRING_LIST.STRING).from(Tables.TEMP_STRING_LIST)