feat(scanner): soft delete series and books

deleted elements will be marked as deleted instead of being removed from the database

komga/src/flyway/resources/db/migration/sqlite/V20210706162229__file_hash.sql

@@ -1,2 +1,8 @@
 ALTER TABLE BOOK
     ADD COLUMN FILE_HASH varchar NOT NULL DEFAULT '';
+
+ALTER TABLE BOOK
+    ADD COLUMN DELETED_DATE datetime NULL DEFAULT NULL;
+
+ALTER TABLE SERIES
+    ADD COLUMN DELETED_DATE datetime NULL DEFAULT NULL;

komga/src/main/kotlin/org/gotson/komga/domain/model/Book.kt

@@ -20,6 +20,8 @@ data class Book(
   val seriesId: String = "",
   val libraryId: String = "",
 
+  val deletedDate: LocalDateTime? = null,
+
   override val createdDate: LocalDateTime = LocalDateTime.now(),
   override val lastModifiedDate: LocalDateTime = LocalDateTime.now()
 ) : Auditable(), Serializable {

komga/src/main/kotlin/org/gotson/komga/domain/model/Series.kt

@@ -16,6 +16,8 @@ data class Series(
   val libraryId: String = "",
   val bookCount: Int = 0,
 
+  val deletedDate: LocalDateTime? = null,
+
   override val createdDate: LocalDateTime = LocalDateTime.now(),
   override val lastModifiedDate: LocalDateTime = LocalDateTime.now()
 ) : Auditable(), Serializable {

komga/src/main/kotlin/org/gotson/komga/domain/service/BookLifecycle.kt

@@ -27,6 +27,7 @@ import org.springframework.transaction.support.TransactionTemplate
 import java.io.File
 import java.nio.file.Files
 import java.nio.file.Paths
+import java.time.LocalDateTime
 
 private val logger = KotlinLogging.logger {}
 
@@ -228,6 +229,14 @@ class BookLifecycle(
     eventPublisher.publishEvent(DomainEvent.BookDeleted(book))
   }
 
+  fun softDeleteMany(books: Collection<Book>) {
+    logger.info { "Soft delete books: $books" }
+    val deletedDate = LocalDateTime.now()
+    bookRepository.update(books.map { it.copy(deletedDate = deletedDate) })
+
+    books.forEach { eventPublisher.publishEvent(DomainEvent.BookUpdated(it)) }
+  }
+
   @Transactional
   fun deleteMany(books: Collection<Book>) {
     val bookIds = books.map { it.id }

komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt

@@ -49,15 +49,15 @@ class LibraryContentLifecycle(
 
       // delete series that don't exist anymore
       if (scannedSeries.isEmpty()) {
-        logger.info { "Scan returned no series, deleting all existing series" }
+        logger.info { "Scan returned no series, soft deleting all existing series" }
         val series = seriesRepository.findAllByLibraryId(library.id)
-        seriesLifecycle.deleteMany(series)
+        seriesLifecycle.softDeleteMany(series)
       } else {
         scannedSeries.keys.map { it.url }.let { urls ->
           val series = seriesRepository.findAllByLibraryIdAndUrlNotIn(library.id, urls)
           if (series.isNotEmpty()) {
-            logger.info { "Deleting series not on disk anymore: $series" }
+            logger.info { "Soft deleting series not on disk anymore: $series" }
-            seriesLifecycle.deleteMany(series)
+            seriesLifecycle.softDeleteMany(series)
           }
         }
       }
@@ -74,10 +74,10 @@ class LibraryContentLifecycle(
         } else {
           // if series already exists, update it
           logger.debug { "Scanned series already exists. Scanned: $newSeries, Existing: $existingSeries" }
-          val seriesChanged = newSeries.fileLastModified.notEquals(existingSeries.fileLastModified)
+          val seriesChanged = newSeries.fileLastModified.notEquals(existingSeries.fileLastModified) || existingSeries.deletedDate != null
           if (seriesChanged) {
             logger.info { "Series changed on disk, updating: $existingSeries" }
-            seriesRepository.update(existingSeries.copy(fileLastModified = newSeries.fileLastModified))
+            seriesRepository.update(existingSeries.copy(fileLastModified = newSeries.fileLastModified, deletedDate = null))
           }
           if (library.scanDeep || seriesChanged) {
             // update list of books with existing entities if they exist
@@ -88,12 +88,13 @@ class LibraryContentLifecycle(
               logger.debug { "Trying to match scanned book by url: $newBook" }
               existingBooks.find { it.url == newBook.url }?.let { existingBook ->
                 logger.debug { "Matched existing book: $existingBook" }
-                if (newBook.fileLastModified.notEquals(existingBook.fileLastModified)) {
+                if (newBook.fileLastModified.notEquals(existingBook.fileLastModified) || existingBook.deletedDate != null) {
                   logger.info { "Book changed on disk, update and reset media status: $existingBook" }
                   val updatedBook = existingBook.copy(
                     fileLastModified = newBook.fileLastModified,
                     fileSize = newBook.fileSize,
                     fileHash = "",
+                    deletedDate = null,
                   )
                   transactionTemplate.executeWithoutResult {
                     mediaRepository.findById(existingBook.id).let {
@@ -111,7 +112,7 @@ class LibraryContentLifecycle(
               .filterNot { existingBook -> newBooksUrls.contains(existingBook.url) }
               .let { books ->
                 logger.info { "Deleting books not on disk anymore: $books" }
-                bookLifecycle.deleteMany(books)
+                bookLifecycle.softDeleteMany(books)
                 books.map { it.seriesId }.distinct().forEach { taskReceiver.refreshSeriesMetadata(it) }
               }
 

komga/src/main/kotlin/org/gotson/komga/domain/service/SeriesLifecycle.kt

@@ -30,6 +30,7 @@ import org.springframework.transaction.annotation.Transactional
 import java.io.File
 import java.nio.file.Files
 import java.nio.file.Paths
+import java.time.LocalDateTime
 
 private val logger = KotlinLogging.logger {}
 private val natSortComparator: Comparator<String> = CaseInsensitiveSimpleNaturalComparator.getInstance()
@@ -136,13 +137,25 @@ class SeriesLifecycle(
     return seriesRepository.findByIdOrNull(series.id)!!
   }
 
+  @Transactional
+  fun softDeleteMany(series: Collection<Series>) {
+    logger.info { "Soft delete series: $series" }
+    val deletedDate = LocalDateTime.now()
+
+    bookLifecycle.softDeleteMany(bookRepository.findAllBySeriesIds(series.map { it.id }))
+    series.forEach {
+      seriesRepository.update(it.copy(deletedDate = deletedDate))
+    }
+
+    series.forEach { eventPublisher.publishEvent(DomainEvent.SeriesUpdated(it)) }
+  }
+
   @Transactional
   fun deleteMany(series: Collection<Series>) {
     val seriesIds = series.map { it.id }
     logger.info { "Delete series ids: $seriesIds" }
 
-    val books = bookRepository.findAllBySeriesIds(seriesIds)
+    bookLifecycle.deleteMany(bookRepository.findAllBySeriesIds(seriesIds))
-    bookLifecycle.deleteMany(books)
 
     readProgressRepository.deleteBySeriesIds(seriesIds)
     collectionRepository.removeSeriesFromAll(seriesIds)

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt

@@ -202,8 +202,9 @@ class BookDao(
           b.FILE_SIZE,
           b.FILE_HASH,
           b.LIBRARY_ID,
-          b.SERIES_ID
+          b.SERIES_ID,
-        ).values(null as String?, null, null, null, null, null, null, null, null)
+          b.DELETED_DATE,
+        ).values(null as String?, null, null, null, null, null, null, null, null, null)
       ).also { step ->
         books.forEach {
           step.bind(
@@ -215,7 +216,8 @@ class BookDao(
             it.fileSize,
             it.fileHash,
             it.libraryId,
-            it.seriesId
+            it.seriesId,
+            it.deletedDate,
           )
         }
       }.execute()
@@ -242,6 +244,7 @@ class BookDao(
       .set(b.FILE_HASH, book.fileHash)
       .set(b.LIBRARY_ID, book.libraryId)
       .set(b.SERIES_ID, book.seriesId)
+      .set(b.DELETED_DATE, book.deletedDate)
       .set(b.LAST_MODIFIED_DATE, LocalDateTime.now(ZoneId.of("Z")))
       .where(b.ID.eq(book.id))
       .execute()
@@ -282,6 +285,7 @@ class BookDao(
       id = id,
       libraryId = libraryId,
       seriesId = seriesId,
+      deletedDate = deletedDate,
       createdDate = createdDate.toCurrentTimeZone(),
       lastModifiedDate = lastModifiedDate.toCurrentTimeZone(),
       number = number

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt

@@ -318,7 +318,8 @@ class BookDtoDao(
       sizeBytes = fileSize,
       media = media,
       metadata = metadata,
-      readProgress = readProgress
+      readProgress = readProgress,
+      deleted = deletedDate != null,
     )
 
   private fun MediaRecord.toDto() =

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt

@@ -90,6 +90,7 @@ class SeriesDao(
       .set(s.URL, series.url.toString())
       .set(s.FILE_LAST_MODIFIED, series.fileLastModified)
       .set(s.LIBRARY_ID, series.libraryId)
+      .set(s.DELETED_DATE, series.deletedDate)
       .execute()
   }
 
@@ -100,6 +101,7 @@ class SeriesDao(
       .set(s.FILE_LAST_MODIFIED, series.fileLastModified)
       .set(s.LIBRARY_ID, series.libraryId)
       .set(s.BOOK_COUNT, series.bookCount)
+      .set(s.DELETED_DATE, series.deletedDate)
       .set(s.LAST_MODIFIED_DATE, LocalDateTime.now(ZoneId.of("Z")))
       .where(s.ID.eq(series.id))
       .execute()
@@ -139,6 +141,7 @@ class SeriesDao(
       id = id,
       libraryId = libraryId,
       bookCount = bookCount,
+      deletedDate = deletedDate,
       createdDate = createdDate.toCurrentTimeZone(),
       lastModifiedDate = lastModifiedDate.toCurrentTimeZone()
     )

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt

@@ -278,6 +278,7 @@ class SeriesDtoDao(
       booksInProgressCount = booksInProgressCount,
       metadata = metadata,
       booksMetadata = booksMetadata,
+      deleted = deletedDate != null,
     )
 
   private fun SeriesMetadataRecord.toDto(genres: Set<String>, tags: Set<String>) =

komga/src/main/kotlin/org/gotson/komga/interfaces/rest/dto/BookDto.kt

@@ -23,7 +23,8 @@ data class BookDto(
   val size: String = BinaryByteUnit.format(sizeBytes),
   val media: MediaDto,
   val metadata: BookMetadataDto,
-  val readProgress: ReadProgressDto? = null
+  val readProgress: ReadProgressDto? = null,
+  val deleted: Boolean,
 )
 
 fun BookDto.restrictUrl(restrict: Boolean) =

komga/src/main/kotlin/org/gotson/komga/interfaces/rest/dto/SeriesDto.kt

@@ -22,6 +22,7 @@ data class SeriesDto(
   val booksInProgressCount: Int,
   val metadata: SeriesMetadataDto,
   val booksMetadata: BookMetadataAggregationDto,
+  val deleted: Boolean,
 )
 
 fun SeriesDto.restrictUrl(restrict: Boolean) =

komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycleTest.kt

@@ -86,7 +86,7 @@ class LibraryContentLifecycleTest(
   }
 
   @Test
-  fun `given existing series when removing files and scanning then only updated Books are persisted`() {
+  fun `given existing series when removing files and scanning then updated Books are persisted and removed books are marked as such`() {
     // given
     val library = makeLibrary()
     libraryRepository.insert(library)
@@ -111,9 +111,41 @@ class LibraryContentLifecycleTest(
     verify(exactly = 2) { mockScanner.scanRootFolder(any()) }
 
     assertThat(allSeries).hasSize(1)
-    assertThat(allBooks).hasSize(1)
+    assertThat(allBooks).hasSize(2)
-    assertThat(allBooks.map { it.name }).containsExactly("book1")
+    assertThat(allBooks.filter { it.deletedDate == null }.map { it.name }).containsExactly("book1")
-    assertThat(bookRepository.count()).describedAs("Orphan book has been removed").isEqualTo(1)
+    assertThat(allBooks.filter { it.deletedDate != null }.map { it.name }).containsExactly("book2")
+  }
+
+  @Test
+  fun `given existing series when removing files and scanning, restoring files and scanning then restored books are available`() {
+    // given
+    val library = makeLibrary()
+    libraryRepository.insert(library)
+
+    val books = listOf(makeBook("book1"), makeBook("book2"))
+    val lessBooks = listOf(makeBook("book1"))
+
+    every { mockScanner.scanRootFolder(any()) }
+      .returnsMany(
+        mapOf(makeSeries(name = "series") to books).toScanResult(),
+        mapOf(makeSeries(name = "series") to lessBooks).toScanResult(),
+        mapOf(makeSeries(name = "series") to books).toScanResult(),
+      )
+    libraryContentLifecycle.scanRootFolder(library) // creation
+    libraryContentLifecycle.scanRootFolder(library) // deletion
+
+    // when
+    libraryContentLifecycle.scanRootFolder(library) // restore
+
+    // then
+    val allSeries = seriesRepository.findAll()
+    val allBooks = bookRepository.findAll().sortedBy { it.number }
+
+    verify(exactly = 3) { mockScanner.scanRootFolder(any()) }
+
+    assertThat(allSeries).hasSize(1)
+    assertThat(allBooks).hasSize(2)
+    assertThat(allBooks.map { it.deletedDate }).containsOnlyNulls()
   }
 
   @Test
@@ -148,7 +180,7 @@ class LibraryContentLifecycleTest(
   }
 
   @Test
-  fun `given existing series when deleting all books and scanning then Series and Books are removed`() {
+  fun `given existing series when deleting all books and scanning then Series and Books are marked as deleted`() {
     // given
     val library = makeLibrary()
     libraryRepository.insert(library)
@@ -166,12 +198,49 @@ class LibraryContentLifecycleTest(
     // then
     verify(exactly = 2) { mockScanner.scanRootFolder(any()) }
 
-    assertThat(seriesRepository.count()).describedAs("Series repository should be empty").isEqualTo(0)
+    val allSeries = seriesRepository.findAll()
-    assertThat(bookRepository.count()).describedAs("Book repository should be empty").isEqualTo(0)
+    val allBooks = bookRepository.findAll()
+
+    assertThat(allSeries.map { it.deletedDate }).doesNotContainNull()
+    assertThat(allSeries).hasSize(1)
+    assertThat(allBooks.map { it.deletedDate }).doesNotContainNull()
+    assertThat(allBooks).hasSize(1)
   }
 
   @Test
-  fun `given existing Series when deleting all books of one series and scanning then series and its Books are removed`() {
+  fun `given existing series when deleting all books and scanning then restoring and scanning then Series and Books are available`() {
+    // given
+    val library = makeLibrary()
+    libraryRepository.insert(library)
+
+    val series = makeSeries(name = "series")
+    val book = makeBook("book1")
+    every { mockScanner.scanRootFolder(any()) }
+      .returnsMany(
+        mapOf(series to listOf(book)).toScanResult(),
+        emptyMap<Series, List<Book>>().toScanResult(),
+        mapOf(series to listOf(book)).toScanResult(),
+      )
+    libraryContentLifecycle.scanRootFolder(library) // creation
+    libraryContentLifecycle.scanRootFolder(library) // deletion
+
+    // when
+    libraryContentLifecycle.scanRootFolder(library) // restore
+
+    // then
+    verify(exactly = 3) { mockScanner.scanRootFolder(any()) }
+
+    val allSeries = seriesRepository.findAll()
+    val allBooks = bookRepository.findAll()
+
+    assertThat(allSeries.map { it.deletedDate }).containsOnlyNulls()
+    assertThat(allSeries).hasSize(1)
+    assertThat(allBooks.map { it.deletedDate }).containsOnlyNulls()
+    assertThat(allBooks).hasSize(1)
+  }
+
+  @Test
+  fun `given existing Series when deleting all books of one series and scanning then series and its Books are marked as deleted`() {
     // given
     val library = makeLibrary()
     libraryRepository.insert(library)
@@ -192,8 +261,20 @@ class LibraryContentLifecycleTest(
     // then
     verify(exactly = 2) { mockScanner.scanRootFolder(any()) }
 
-    assertThat(seriesRepository.count()).describedAs("Series repository should not be empty").isEqualTo(1)
+    val (series, deletedSeries) = seriesRepository.findAll().partition { it.deletedDate == null }
-    assertThat(bookRepository.count()).describedAs("Book repository should not be empty").isEqualTo(1)
+    val (books, deletedBooks) = bookRepository.findAll().partition { it.deletedDate == null }
+
+    assertThat(series).hasSize(1)
+    assertThat(series.map { it.name }).containsExactlyInAnyOrder("series")
+
+    assertThat(deletedSeries).hasSize(1)
+    assertThat(deletedSeries.map { it.name }).containsExactlyInAnyOrder("series2")
+
+    assertThat(books).hasSize(1)
+    assertThat(books.map { it.name }).containsExactlyInAnyOrder("book1")
+
+    assertThat(deletedBooks).hasSize(1)
+    assertThat(deletedBooks.map { it.name }).containsExactlyInAnyOrder("book2")
   }
 
   @Test
@@ -294,8 +375,8 @@ class LibraryContentLifecycleTest(
     libraryContentLifecycle.scanRootFolder(library1)
     libraryContentLifecycle.scanRootFolder(library2)
 
-    assertThat(seriesRepository.count()).describedAs("Series repository should be empty").isEqualTo(2)
+    assertThat(seriesRepository.count()).describedAs("Series repository should not be empty").isEqualTo(2)
-    assertThat(bookRepository.count()).describedAs("Book repository should be empty").isEqualTo(2)
+    assertThat(bookRepository.count()).describedAs("Book repository should not be empty").isEqualTo(2)
 
     // when
     libraryContentLifecycle.scanRootFolder(library2)
@@ -304,7 +385,19 @@ class LibraryContentLifecycleTest(
     verify(exactly = 1) { mockScanner.scanRootFolder(Paths.get(library1.root.toURI())) }
     verify(exactly = 2) { mockScanner.scanRootFolder(Paths.get(library2.root.toURI())) }
 
-    assertThat(seriesRepository.count()).describedAs("Series repository should be empty").isEqualTo(1)
+    val (seriesLib1, seriesLib2) = seriesRepository.findAll().partition { it.libraryId == library1.id }
-    assertThat(bookRepository.count()).describedAs("Book repository should be empty").isEqualTo(1)
+    val (booksLib1, booksLib2) = bookRepository.findAll().partition { it.libraryId == library1.id }
+
+    assertThat(seriesLib1.map { it.deletedDate }).containsOnlyNulls()
+    assertThat(seriesLib1.map { it.name }).containsExactlyInAnyOrder("series1")
+
+    assertThat(seriesLib2.map { it.deletedDate }).doesNotContainNull()
+    assertThat(seriesLib2.map { it.name }).containsExactlyInAnyOrder("series2")
+
+    assertThat(booksLib1.map { it.deletedDate }).containsOnlyNulls()
+    assertThat(booksLib1.map { it.name }).containsExactlyInAnyOrder("book1")
+
+    assertThat(booksLib2.map { it.deletedDate }).doesNotContainNull()
+    assertThat(booksLib2.map { it.name }).containsExactlyInAnyOrder("book2")
   }
 }

komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/BookDaoTest.kt

@@ -58,7 +58,8 @@ class BookDaoTest(
       fileSize = 3,
       fileHash = "abc",
       seriesId = series.id,
-      libraryId = library.id
+      libraryId = library.id,
+      deletedDate = LocalDateTime.now(),
     )
 
     bookDao.insert(book)
@@ -72,6 +73,7 @@ class BookDaoTest(
     assertThat(created.fileLastModified).isEqualToIgnoringNanos(book.fileLastModified)
     assertThat(created.fileSize).isEqualTo(book.fileSize)
     assertThat(created.fileHash).isEqualTo(book.fileHash)
+    assertThat(created.deletedDate).isEqualTo(book.deletedDate)
   }
 
   @Test
@@ -82,7 +84,7 @@ class BookDaoTest(
       fileLastModified = LocalDateTime.now(),
       fileSize = 3,
       seriesId = series.id,
-      libraryId = library.id
+      libraryId = library.id,
     )
     bookDao.insert(book)
 
@@ -95,6 +97,7 @@ class BookDaoTest(
         fileLastModified = modificationDate,
         fileSize = 5,
         fileHash = "def",
+        deletedDate = LocalDateTime.now(),
       )
     }
 
@@ -111,6 +114,7 @@ class BookDaoTest(
     assertThat(modified.fileLastModified).isEqualToIgnoringNanos(modificationDate)
     assertThat(modified.fileSize).isEqualTo(5)
     assertThat(modified.fileHash).isEqualTo("def")
+    assertThat(modified.deletedDate).isEqualToIgnoringNanos(updated.deletedDate)
   }
 
   @Test

komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDaoTest.kt

@@ -48,7 +48,8 @@ class SeriesDaoTest(
       name = "Series",
       url = URL("file://series"),
       fileLastModified = now,
-      libraryId = library.id
+      libraryId = library.id,
+      deletedDate = now,
     )
 
     seriesDao.insert(series)
@@ -60,6 +61,44 @@ class SeriesDaoTest(
     assertThat(created.name).isEqualTo(series.name)
     assertThat(created.url).isEqualTo(series.url)
     assertThat(created.fileLastModified).isEqualToIgnoringNanos(series.fileLastModified)
+    assertThat(created.deletedDate).isEqualToIgnoringNanos(series.deletedDate)
+  }
+
+  @Test
+  fun `given a series when updating then it is persisted`() {
+    val now = LocalDateTime.now()
+    val series = Series(
+      name = "Series",
+      url = URL("file://series"),
+      fileLastModified = now,
+      libraryId = library.id,
+    )
+
+    seriesDao.insert(series)
+
+    val modificationDate = LocalDateTime.now()
+
+    val updated = seriesDao.findByIdOrNull(series.id)!!.copy(
+      name = "Updated",
+      url = URL("file://updated"),
+      fileLastModified = modificationDate,
+      bookCount = 5,
+      deletedDate = LocalDateTime.now(),
+    )
+
+    seriesDao.update(updated)
+    val modified = seriesDao.findByIdOrNull(updated.id)!!
+
+    assertThat(modified.id).isEqualTo(updated.id)
+    assertThat(modified.createdDate).isEqualTo(updated.createdDate)
+    assertThat(modified.lastModifiedDate)
+      .isCloseTo(modificationDate, offset)
+      .isNotEqualTo(updated.lastModifiedDate)
+    assertThat(modified.name).isEqualTo("Updated")
+    assertThat(modified.url).isEqualTo(URL("file://updated"))
+    assertThat(modified.fileLastModified).isEqualToIgnoringNanos(modificationDate)
+    assertThat(modified.bookCount).isEqualTo(5)
+    assertThat(modified.deletedDate).isEqualToIgnoringNanos(updated.deletedDate)
   }
 
   @Test