diff --git a/komga/src/flyway/resources/db/migration/sqlite/V20210831113524__temp_table_urls.sql b/komga/src/flyway/resources/db/migration/sqlite/V20210831113524__temp_table_urls.sql new file mode 100644 index 000000000..1f32483f4 --- /dev/null +++ b/komga/src/flyway/resources/db/migration/sqlite/V20210831113524__temp_table_urls.sql @@ -0,0 +1,4 @@ +CREATE TABLE TEMP_URL_LIST +( + URL varchar NOT NULL +); diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt index 8271624c2..c9ef439b5 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt @@ -14,7 +14,7 @@ interface BookRepository { fun findAll(): Collection fun findAllBySeriesId(seriesId: String): Collection fun findAllBySeriesIds(seriesIds: Collection): Collection - fun findAllByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection): Collection + fun findAllNotDeletedByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection): Collection fun findAll(bookSearch: BookSearch): Collection fun findAll(bookSearch: BookSearch, pageable: Pageable): Page fun findAllDeletedByFileSize(fileSize: Long): Collection diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesRepository.kt index 0bc6da220..75b55994a 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesRepository.kt @@ -10,7 +10,7 @@ interface SeriesRepository { fun findAll(): Collection fun findAllByLibraryId(libraryId: String): Collection - fun findAllByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection): Collection + fun findAllNotDeletedByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection): Collection fun findAllByTitle(title: String): Collection fun findAll(search: SeriesSearch): Collection diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt index eeac5814a..7f372dc6a 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt @@ -96,7 +96,7 @@ class LibraryContentLifecycle( seriesLifecycle.softDeleteMany(series) } else { scannedSeries.keys.map { it.url }.let { urls -> - val series = seriesRepository.findAllByLibraryIdAndUrlNotIn(library.id, urls).filterNot { it.deletedDate != null } + val series = seriesRepository.findAllNotDeletedByLibraryIdAndUrlNotIn(library.id, urls) if (series.isNotEmpty()) { logger.info { "Soft deleting series not on disk anymore: $series" } seriesLifecycle.softDeleteMany(series) @@ -106,7 +106,7 @@ class LibraryContentLifecycle( // delete books that don't exist anymore. We need to do this now, so trash bin can work val seriesToSortAndRefresh = scannedSeries.values.flatten().map { it.url }.let { urls -> - val books = bookRepository.findAllByLibraryIdAndUrlNotIn(library.id, urls).filterNot { it.deletedDate != null } + val books = bookRepository.findAllNotDeletedByLibraryIdAndUrlNotIn(library.id, urls) if (books.isNotEmpty()) { logger.info { "Soft deleting books not on disk anymore: $books" } bookLifecycle.softDeleteMany(books) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt index 2c6138bb4..131647593 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt @@ -28,6 +28,7 @@ class BookDao( private val m = Tables.MEDIA private val d = Tables.BOOK_METADATA private val r = Tables.READ_PROGRESS + private val u = Tables.TEMP_URL_LIST private val sorts = mapOf( "createdDate" to b.CREATED_DATE, @@ -62,11 +63,26 @@ class BookDao( .fetchInto(b) .map { it.toDomain() } - override fun findAllByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection): Collection = - dsl.selectFrom(b) - .where(b.LIBRARY_ID.eq(libraryId).and(b.URL.notIn(urls.map { it.toString() }))) + @Transactional + override fun findAllNotDeletedByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection): Collection { + // insert urls in a temporary table, else the select size can exceed the statement limit + dsl.deleteFrom(u).execute() + + dsl.batch( + dsl.insertInto(u, u.URL).values(null as String?) + ).also { step -> + urls.forEach { + step.bind(it.toString()) + } + }.execute() + + return dsl.selectFrom(b) + .where(b.LIBRARY_ID.eq(libraryId)) + .and(b.DELETED_DATE.isNull) + .and(b.URL.notIn(dsl.selectFrom(u))) .fetchInto(b) .map { it.toDomain() } + } override fun findAllDeletedByFileSize(fileSize: Long): Collection = dsl.selectFrom(b) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt index ce4d9b59c..d18ed620b 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt @@ -9,6 +9,7 @@ import org.jooq.Condition import org.jooq.DSLContext import org.jooq.impl.DSL import org.springframework.stereotype.Component +import org.springframework.transaction.annotation.Transactional import java.net.URL import java.time.LocalDateTime import java.time.ZoneId @@ -21,6 +22,7 @@ class SeriesDao( private val s = Tables.SERIES private val d = Tables.SERIES_METADATA private val cs = Tables.COLLECTION_SERIES + private val u = Tables.TEMP_URL_LIST override fun findAll(): Collection = dsl.selectFrom(s) @@ -39,11 +41,26 @@ class SeriesDao( .fetchInto(s) .map { it.toDomain() } - override fun findAllByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection): List = - dsl.selectFrom(s) - .where(s.LIBRARY_ID.eq(libraryId).and(s.URL.notIn(urls.map { it.toString() }))) + @Transactional + override fun findAllNotDeletedByLibraryIdAndUrlNotIn(libraryId: String, urls: Collection): List { + // insert urls in a temporary table, else the select size can exceed the statement limit + dsl.deleteFrom(u).execute() + + dsl.batch( + dsl.insertInto(u, u.URL).values(null as String?) + ).also { step -> + urls.forEach { + step.bind(it.toString()) + } + }.execute() + + return dsl.selectFrom(s) + .where(s.LIBRARY_ID.eq(libraryId)) + .and(s.DELETED_DATE.isNull) + .and(s.URL.notIn(dsl.selectFrom(u))) .fetchInto(s) .map { it.toDomain() } + } override fun findByLibraryIdAndUrlOrNull(libraryId: String, url: URL): Series? = dsl.selectFrom(s) diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDaoTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDaoTest.kt index 51a22d0ca..31561d4eb 100644 --- a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDaoTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDaoTest.kt @@ -269,8 +269,8 @@ class SeriesDaoTest( ) seriesDao.insert(series) - val found = seriesDao.findAllByLibraryIdAndUrlNotIn(library.id, listOf(URL("file://series2"))) - val notFound = seriesDao.findAllByLibraryIdAndUrlNotIn(library.id, listOf(URL("file://series"))) + val found = seriesDao.findAllNotDeletedByLibraryIdAndUrlNotIn(library.id, listOf(URL("file://series2"))) + val notFound = seriesDao.findAllNotDeletedByLibraryIdAndUrlNotIn(library.id, listOf(URL("file://series"))) assertThat(found).hasSize(1) assertThat(found.first().name).isEqualTo("Series")