From 871ec60869fe57d2c823b79006a6f5d38c5b9372 Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Mon, 16 Aug 2021 14:41:56 +0800 Subject: [PATCH] fix(scanner): fail scan if root folder is unavailable this will prevent soft deleting the whole library, and rehash everything when available again closes #617 --- .../V20210816113108__library_unavailable.sql | 2 + .../org/gotson/komga/domain/model/Library.kt | 2 + .../domain/service/LibraryContentLifecycle.kt | 90 +++++++++++-------- .../komga/infrastructure/jooq/LibraryDao.kt | 3 + .../komga/interfaces/rest/dto/LibraryDto.kt | 2 + .../service/LibraryContentLifecycleTest.kt | 19 ++-- 6 files changed, 72 insertions(+), 46 deletions(-) create mode 100644 komga/src/flyway/resources/db/migration/sqlite/V20210816113108__library_unavailable.sql diff --git a/komga/src/flyway/resources/db/migration/sqlite/V20210816113108__library_unavailable.sql b/komga/src/flyway/resources/db/migration/sqlite/V20210816113108__library_unavailable.sql new file mode 100644 index 000000000..836017645 --- /dev/null +++ b/komga/src/flyway/resources/db/migration/sqlite/V20210816113108__library_unavailable.sql @@ -0,0 +1,2 @@ +ALTER TABLE LIBRARY + ADD COLUMN UNAVAILABLE_DATE datetime NULL DEFAULT NULL; diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt index 021757c3f..a68f3d690 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt @@ -26,6 +26,8 @@ data class Library( val emptyTrashAfterScan: Boolean = false, val seriesCover: SeriesCover = SeriesCover.FIRST, + val unavailableDate: LocalDateTime? = null, + val id: String = TsidCreator.getTsid256().toString(), override val createdDate: LocalDateTime = LocalDateTime.now(), diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt index fed903d84..eeac5814a 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt @@ -1,20 +1,22 @@ package org.gotson.komga.domain.service import mu.KotlinLogging +import org.gotson.komga.application.events.EventPublisher import org.gotson.komga.application.tasks.TaskReceiver import org.gotson.komga.domain.model.Book import org.gotson.komga.domain.model.BookMetadataPatchCapability import org.gotson.komga.domain.model.BookSearch import org.gotson.komga.domain.model.DirectoryNotFoundException +import org.gotson.komga.domain.model.DomainEvent import org.gotson.komga.domain.model.Library import org.gotson.komga.domain.model.Media -import org.gotson.komga.domain.model.ScanResult import org.gotson.komga.domain.model.Series import org.gotson.komga.domain.model.SeriesSearch import org.gotson.komga.domain.model.Sidecar import org.gotson.komga.domain.model.ThumbnailBook import org.gotson.komga.domain.persistence.BookMetadataRepository import org.gotson.komga.domain.persistence.BookRepository +import org.gotson.komga.domain.persistence.LibraryRepository import org.gotson.komga.domain.persistence.MediaRepository import org.gotson.komga.domain.persistence.ReadListRepository import org.gotson.komga.domain.persistence.ReadProgressRepository @@ -30,6 +32,7 @@ import org.gotson.komga.infrastructure.language.toIndexedMap import org.springframework.stereotype.Service import org.springframework.transaction.support.TransactionTemplate import java.nio.file.Paths +import java.time.LocalDateTime import kotlin.time.measureTime private val logger = KotlinLogging.logger {} @@ -39,6 +42,7 @@ class LibraryContentLifecycle( private val fileSystemScanner: FileSystemScanner, private val seriesRepository: SeriesRepository, private val bookRepository: BookRepository, + private val libraryRepository: LibraryRepository, private val bookLifecycle: BookLifecycle, private val mediaRepository: MediaRepository, private val seriesLifecycle: SeriesLifecycle, @@ -55,15 +59,27 @@ class LibraryContentLifecycle( private val readProgressRepository: ReadProgressRepository, private val collectionRepository: SeriesCollectionRepository, private val thumbnailBookRepository: ThumbnailBookRepository, + private val eventPublisher: EventPublisher, ) { fun scanRootFolder(library: Library) { logger.info { "Updating library: $library" } measureTime { - val (scanResult, rootFolderInaccessible) = try { - fileSystemScanner.scanRootFolder(Paths.get(library.root.toURI()), library.scanForceModifiedTime) to false + val scanResult = try { + fileSystemScanner.scanRootFolder(Paths.get(library.root.toURI()), library.scanForceModifiedTime) } catch (e: DirectoryNotFoundException) { - ScanResult(emptyMap(), emptyList()) to true + library.copy(unavailableDate = LocalDateTime.now()).let { + libraryRepository.update(it) + eventPublisher.publishEvent(DomainEvent.LibraryUpdated(it)) + } + throw e + } + + if (library.unavailableDate != null) { + library.copy(unavailableDate = null).let { + libraryRepository.update(it) + eventPublisher.publishEvent(DomainEvent.LibraryUpdated(it)) + } } val scannedSeries = @@ -164,45 +180,43 @@ class LibraryContentLifecycle( taskReceiver.refreshSeriesMetadata(it.id) } - if (!rootFolderInaccessible) { - val existingSidecars = sidecarRepository.findAll() - scanResult.sidecars.forEach { newSidecar -> - val existingSidecar = existingSidecars.firstOrNull { it.url == newSidecar.url } - if (existingSidecar == null || existingSidecar.lastModifiedTime.notEquals(newSidecar.lastModifiedTime)) { - when (newSidecar.source) { - Sidecar.Source.SERIES -> - seriesRepository.findByLibraryIdAndUrlOrNull(library.id, newSidecar.parentUrl)?.let { series -> - logger.info { "Sidecar changed on disk (${newSidecar.url}, refresh Series for ${newSidecar.type}: $series" } - when (newSidecar.type) { - Sidecar.Type.ARTWORK -> taskReceiver.refreshSeriesLocalArtwork(series.id) - Sidecar.Type.METADATA -> taskReceiver.refreshSeriesMetadata(series.id) - } + val existingSidecars = sidecarRepository.findAll() + scanResult.sidecars.forEach { newSidecar -> + val existingSidecar = existingSidecars.firstOrNull { it.url == newSidecar.url } + if (existingSidecar == null || existingSidecar.lastModifiedTime.notEquals(newSidecar.lastModifiedTime)) { + when (newSidecar.source) { + Sidecar.Source.SERIES -> + seriesRepository.findByLibraryIdAndUrlOrNull(library.id, newSidecar.parentUrl)?.let { series -> + logger.info { "Sidecar changed on disk (${newSidecar.url}, refresh Series for ${newSidecar.type}: $series" } + when (newSidecar.type) { + Sidecar.Type.ARTWORK -> taskReceiver.refreshSeriesLocalArtwork(series.id) + Sidecar.Type.METADATA -> taskReceiver.refreshSeriesMetadata(series.id) } - Sidecar.Source.BOOK -> - bookRepository.findByLibraryIdAndUrlOrNull(library.id, newSidecar.parentUrl)?.let { book -> - logger.info { "Sidecar changed on disk (${newSidecar.url}, refresh Book for ${newSidecar.type}: $book" } - when (newSidecar.type) { - Sidecar.Type.ARTWORK -> taskReceiver.refreshBookLocalArtwork(book.id) - Sidecar.Type.METADATA -> taskReceiver.refreshBookMetadata(book.id) - } + } + Sidecar.Source.BOOK -> + bookRepository.findByLibraryIdAndUrlOrNull(library.id, newSidecar.parentUrl)?.let { book -> + logger.info { "Sidecar changed on disk (${newSidecar.url}, refresh Book for ${newSidecar.type}: $book" } + when (newSidecar.type) { + Sidecar.Type.ARTWORK -> taskReceiver.refreshBookLocalArtwork(book.id) + Sidecar.Type.METADATA -> taskReceiver.refreshBookMetadata(book.id) } - } - sidecarRepository.save(library.id, newSidecar) + } } + sidecarRepository.save(library.id, newSidecar) } - - // cleanup sidecars that don't exist anymore - scanResult.sidecars.map { it.url }.let { newSidecarsUrls -> - existingSidecars - .filterNot { existing -> newSidecarsUrls.contains(existing.url) } - .let { sidecars -> - sidecarRepository.deleteByLibraryIdAndUrls(library.id, sidecars.map { it.url }) - } - } - - if (library.emptyTrashAfterScan) emptyTrash(library) - else cleanupEmptySets() } + + // cleanup sidecars that don't exist anymore + scanResult.sidecars.map { it.url }.let { newSidecarsUrls -> + existingSidecars + .filterNot { existing -> newSidecarsUrls.contains(existing.url) } + .let { sidecars -> + sidecarRepository.deleteByLibraryIdAndUrls(library.id, sidecars.map { it.url }) + } + } + + if (library.emptyTrashAfterScan) emptyTrash(library) + else cleanupEmptySets() }.also { logger.info { "Library updated in $it" } } } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt index 192b7a2b6..5444fb9fe 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt @@ -75,6 +75,7 @@ class LibraryDao( .set(l.REPAIR_EXTENSIONS, library.repairExtensions) .set(l.CONVERT_TO_CBZ, library.convertToCbz) .set(l.EMPTY_TRASH_AFTER_SCAN, library.emptyTrashAfterScan) + .set(l.UNAVAILABLE_DATE, library.unavailableDate) .set(l.SERIES_COVER, library.seriesCover.toString()) .execute() } @@ -99,6 +100,7 @@ class LibraryDao( .set(l.CONVERT_TO_CBZ, library.convertToCbz) .set(l.EMPTY_TRASH_AFTER_SCAN, library.emptyTrashAfterScan) .set(l.SERIES_COVER, library.seriesCover.toString()) + .set(l.UNAVAILABLE_DATE, library.unavailableDate) .set(l.LAST_MODIFIED_DATE, LocalDateTime.now(ZoneId.of("Z"))) .where(l.ID.eq(library.id)) .execute() @@ -125,6 +127,7 @@ class LibraryDao( convertToCbz = convertToCbz, emptyTrashAfterScan = emptyTrashAfterScan, seriesCover = Library.SeriesCover.valueOf(seriesCover), + unavailableDate = unavailableDate, id = id, createdDate = createdDate.toCurrentTimeZone(), lastModifiedDate = lastModifiedDate.toCurrentTimeZone() diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/dto/LibraryDto.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/dto/LibraryDto.kt index 2776d0d75..ec8b9924d 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/dto/LibraryDto.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/dto/LibraryDto.kt @@ -22,6 +22,7 @@ data class LibraryDto( val convertToCbz: Boolean, val emptyTrashAfterScan: Boolean, val seriesCover: SeriesCoverDto, + val unavailable: Boolean, ) fun Library.toDto(includeRoot: Boolean) = LibraryDto( @@ -43,4 +44,5 @@ fun Library.toDto(includeRoot: Boolean) = LibraryDto( convertToCbz = convertToCbz, emptyTrashAfterScan = emptyTrashAfterScan, seriesCover = seriesCover.toDto(), + unavailable = unavailableDate != null, ) diff --git a/komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycleTest.kt b/komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycleTest.kt index 8e8e4d0b3..0b636ffe7 100644 --- a/komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycleTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycleTest.kt @@ -7,6 +7,7 @@ import io.mockk.just import io.mockk.slot import io.mockk.verify import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.catchThrowable import org.gotson.komga.application.tasks.TaskReceiver import org.gotson.komga.domain.model.Book import org.gotson.komga.domain.model.BookMetadataPatchCapability @@ -429,7 +430,7 @@ class LibraryContentLifecycleTest( } @Test - fun `given library with auto empty trash when scanning and the root folder is not accessible then trash is not emptied automatically`() { + fun `given library when scanning and the root folder is not accessible then exception is thrown`() { // given val library = makeLibrary().copy(emptyTrashAfterScan = true) libraryRepository.insert(library) @@ -442,7 +443,7 @@ class LibraryContentLifecycleTest( libraryContentLifecycle.scanRootFolder(library) // when - libraryContentLifecycle.scanRootFolder(library) + val thrown = catchThrowable { libraryContentLifecycle.scanRootFolder(library) } // then verify(exactly = 2) { mockScanner.scanRootFolder(any()) } @@ -450,15 +451,17 @@ class LibraryContentLifecycleTest( val (series, deletedSeries) = seriesRepository.findAll().partition { it.deletedDate == null } val (books, deletedBooks) = bookRepository.findAll().partition { it.deletedDate == null } - assertThat(series).isEmpty() + assertThat(series).hasSize(2) + assertThat(series.map { it.name }).containsExactlyInAnyOrder("series", "series2") - assertThat(deletedSeries).hasSize(2) - assertThat(deletedSeries.map { it.name }).containsExactlyInAnyOrder("series", "series2") + assertThat(deletedSeries).isEmpty() - assertThat(books).isEmpty() + assertThat(books).hasSize(3) + assertThat(books.map { it.name }).containsExactlyInAnyOrder("book1", "book2", "book3") - assertThat(deletedBooks).hasSize(3) - assertThat(deletedBooks.map { it.name }).containsExactlyInAnyOrder("book1", "book2", "book3") + assertThat(deletedBooks).isEmpty() + + assertThat(thrown).isExactlyInstanceOf(DirectoryNotFoundException::class.java) } }