Backup_Repos/komga
1
0
Fork 0
mirror of https://github.com/gotson/komga.git synced 2025-12-20 15:34:17 +01:00
Code Issues Projects Releases Wiki Activity

feat(scanner): soft delete series and books

Browse source 
deleted elements will be marked as deleted instead of being removed from the database
This commit is contained in:
Gauthier Roebroeck 2021-07-07 15:57:45 +08:00
parent 7ad738a645
commit f0664e9791
15 changed files with 212 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
komga/src/flyway/resources/db/migration/sqlite/V20210706162229__file_hash.sql
View file

@ -1,2 +1,8 @@
ALTER TABLE BOOK
ADD COLUMN FILE_HASH varchar NOT NULL DEFAULT '';
ALTER TABLE BOOK
ADD COLUMN DELETED_DATE datetime NULL DEFAULT NULL;
ALTER TABLE SERIES
ADD COLUMN DELETED_DATE datetime NULL DEFAULT NULL;

2
komga/src/main/kotlin/org/gotson/komga/domain/model/Book.kt
View file

@ -20,6 +20,8 @@ data class Book(
val seriesId: String = "",
val libraryId: String = "",
val deletedDate: LocalDateTime? = null,
override val createdDate: LocalDateTime = LocalDateTime.now(),
override val lastModifiedDate: LocalDateTime = LocalDateTime.now()
) : Auditable(), Serializable {

2
komga/src/main/kotlin/org/gotson/komga/domain/model/Series.kt
View file

@ -16,6 +16,8 @@ data class Series(
val libraryId: String = "",
val bookCount: Int = 0,
val deletedDate: LocalDateTime? = null,
override val createdDate: LocalDateTime = LocalDateTime.now(),
override val lastModifiedDate: LocalDateTime = LocalDateTime.now()
) : Auditable(), Serializable {

9
komga/src/main/kotlin/org/gotson/komga/domain/service/BookLifecycle.kt
View file

@ -27,6 +27,7 @@ import org.springframework.transaction.support.TransactionTemplate
import java.io.File
import java.nio.file.Files
import java.nio.file.Paths
import java.time.LocalDateTime
private val logger = KotlinLogging.logger {}
@ -228,6 +229,14 @@ class BookLifecycle(
eventPublisher.publishEvent(DomainEvent.BookDeleted(book))
}
fun softDeleteMany(books: Collection<Book>) {
logger.info { "Soft delete books: $books" }
val deletedDate = LocalDateTime.now()
bookRepository.update(books.map { it.copy(deletedDate = deletedDate) })
books.forEach { eventPublisher.publishEvent(DomainEvent.BookUpdated(it)) }
}
@Transactional
fun deleteMany(books: Collection<Book>) {
val bookIds = books.map { it.id }

17
komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt
View file

@ -49,15 +49,15 @@ class LibraryContentLifecycle(
// delete series that don't exist anymore
if (scannedSeries.isEmpty()) {
logger.info { "Scan returned no series, deleting all existing series" }
logger.info { "Scan returned no series, soft deleting all existing series" }
val series = seriesRepository.findAllByLibraryId(library.id)
seriesLifecycle.deleteMany(series)
seriesLifecycle.softDeleteMany(series)
} else {
scannedSeries.keys.map { it.url }.let { urls ->
val series = seriesRepository.findAllByLibraryIdAndUrlNotIn(library.id, urls)
if (series.isNotEmpty()) {
logger.info { "Deleting series not on disk anymore: $series" }
seriesLifecycle.deleteMany(series)
logger.info { "Soft deleting series not on disk anymore: $series" }
seriesLifecycle.softDeleteMany(series)
}
}
}
@ -74,10 +74,10 @@ class LibraryContentLifecycle(
} else {
// if series already exists, update it
logger.debug { "Scanned series already exists. Scanned: $newSeries, Existing: $existingSeries" }
val seriesChanged = newSeries.fileLastModified.notEquals(existingSeries.fileLastModified)
val seriesChanged = newSeries.fileLastModified.notEquals(existingSeries.fileLastModified) || existingSeries.deletedDate != null
if (seriesChanged) {
logger.info { "Series changed on disk, updating: $existingSeries" }
seriesRepository.update(existingSeries.copy(fileLastModified = newSeries.fileLastModified))
seriesRepository.update(existingSeries.copy(fileLastModified = newSeries.fileLastModified, deletedDate = null))
}
if (library.scanDeep || seriesChanged) {
// update list of books with existing entities if they exist
@ -88,12 +88,13 @@ class LibraryContentLifecycle(
logger.debug { "Trying to match scanned book by url: $newBook" }
existingBooks.find { it.url == newBook.url }?.let { existingBook ->
logger.debug { "Matched existing book: $existingBook" }
if (newBook.fileLastModified.notEquals(existingBook.fileLastModified)) {
if (newBook.fileLastModified.notEquals(existingBook.fileLastModified) || existingBook.deletedDate != null) {
logger.info { "Book changed on disk, update and reset media status: $existingBook" }
val updatedBook = existingBook.copy(
fileLastModified = newBook.fileLastModified,
fileSize = newBook.fileSize,
fileHash = "",
deletedDate = null,
)
transactionTemplate.executeWithoutResult {
mediaRepository.findById(existingBook.id).let {
@ -111,7 +112,7 @@ class LibraryContentLifecycle(
.filterNot { existingBook -> newBooksUrls.contains(existingBook.url) }
.let { books ->
logger.info { "Deleting books not on disk anymore: $books" }
bookLifecycle.deleteMany(books)
bookLifecycle.softDeleteMany(books)
books.map { it.seriesId }.distinct().forEach { taskReceiver.refreshSeriesMetadata(it) }
}

17
komga/src/main/kotlin/org/gotson/komga/domain/service/SeriesLifecycle.kt
View file

@ -30,6 +30,7 @@ import org.springframework.transaction.annotation.Transactional
import java.io.File
import java.nio.file.Files
import java.nio.file.Paths
import java.time.LocalDateTime
private val logger = KotlinLogging.logger {}
private val natSortComparator: Comparator<String> = CaseInsensitiveSimpleNaturalComparator.getInstance()
@ -136,13 +137,25 @@ class SeriesLifecycle(
return seriesRepository.findByIdOrNull(series.id)!!
}
@Transactional
fun softDeleteMany(series: Collection<Series>) {
logger.info { "Soft delete series: $series" }
val deletedDate = LocalDateTime.now()
bookLifecycle.softDeleteMany(bookRepository.findAllBySeriesIds(series.map { it.id }))
series.forEach {
seriesRepository.update(it.copy(deletedDate = deletedDate))
}
series.forEach { eventPublisher.publishEvent(DomainEvent.SeriesUpdated(it)) }
}
@Transactional
fun deleteMany(series: Collection<Series>) {
val seriesIds = series.map { it.id }
logger.info { "Delete series ids: $seriesIds" }
val books = bookRepository.findAllBySeriesIds(seriesIds)
bookLifecycle.deleteMany(books)
bookLifecycle.deleteMany(bookRepository.findAllBySeriesIds(seriesIds))
readProgressRepository.deleteBySeriesIds(seriesIds)
collectionRepository.removeSeriesFromAll(seriesIds)

10
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt
View file

@ -202,8 +202,9 @@ class BookDao(
b.FILE_SIZE,
b.FILE_HASH,
b.LIBRARY_ID,
b.SERIES_ID
).values(null as String?, null, null, null, null, null, null, null, null)
b.SERIES_ID,
b.DELETED_DATE,
).values(null as String?, null, null, null, null, null, null, null, null, null)
).also { step ->
books.forEach {
step.bind(
@ -215,7 +216,8 @@ class BookDao(
it.fileSize,
it.fileHash,
it.libraryId,
it.seriesId
it.seriesId,
it.deletedDate,
)
}
}.execute()
@ -242,6 +244,7 @@ class BookDao(
.set(b.FILE_HASH, book.fileHash)
.set(b.LIBRARY_ID, book.libraryId)
.set(b.SERIES_ID, book.seriesId)
.set(b.DELETED_DATE, book.deletedDate)
.set(b.LAST_MODIFIED_DATE, LocalDateTime.now(ZoneId.of("Z")))
.where(b.ID.eq(book.id))
.execute()
@ -282,6 +285,7 @@ class BookDao(
id = id,
libraryId = libraryId,
seriesId = seriesId,
deletedDate = deletedDate,
createdDate = createdDate.toCurrentTimeZone(),
lastModifiedDate = lastModifiedDate.toCurrentTimeZone(),
number = number

3
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt
View file

@ -318,7 +318,8 @@ class BookDtoDao(
sizeBytes = fileSize,
media = media,
metadata = metadata,
readProgress = readProgress
readProgress = readProgress,
deleted = deletedDate != null,
)
private fun MediaRecord.toDto() =

3
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt
View file

@ -90,6 +90,7 @@ class SeriesDao(
.set(s.URL, series.url.toString())
.set(s.FILE_LAST_MODIFIED, series.fileLastModified)
.set(s.LIBRARY_ID, series.libraryId)
.set(s.DELETED_DATE, series.deletedDate)
.execute()
}
@ -100,6 +101,7 @@ class SeriesDao(
.set(s.FILE_LAST_MODIFIED, series.fileLastModified)
.set(s.LIBRARY_ID, series.libraryId)
.set(s.BOOK_COUNT, series.bookCount)
.set(s.DELETED_DATE, series.deletedDate)
.set(s.LAST_MODIFIED_DATE, LocalDateTime.now(ZoneId.of("Z")))
.where(s.ID.eq(series.id))
.execute()
@ -139,6 +141,7 @@ class SeriesDao(
id = id,
libraryId = libraryId,
bookCount = bookCount,
deletedDate = deletedDate,
createdDate = createdDate.toCurrentTimeZone(),
lastModifiedDate = lastModifiedDate.toCurrentTimeZone()
)

1
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt
View file

@ -278,6 +278,7 @@ class SeriesDtoDao(
booksInProgressCount = booksInProgressCount,
metadata = metadata,
booksMetadata = booksMetadata,
deleted = deletedDate != null,
)
private fun SeriesMetadataRecord.toDto(genres: Set<String>, tags: Set<String>) =

3
komga/src/main/kotlin/org/gotson/komga/interfaces/rest/dto/BookDto.kt
View file

@ -23,7 +23,8 @@ data class BookDto(
val size: String = BinaryByteUnit.format(sizeBytes),
val media: MediaDto,
val metadata: BookMetadataDto,
val readProgress: ReadProgressDto? = null
val readProgress: ReadProgressDto? = null,
val deleted: Boolean,
)
fun BookDto.restrictUrl(restrict: Boolean) =

1
komga/src/main/kotlin/org/gotson/komga/interfaces/rest/dto/SeriesDto.kt
View file

@ -22,6 +22,7 @@ data class SeriesDto(
val booksInProgressCount: Int,
val metadata: SeriesMetadataDto,
val booksMetadata: BookMetadataAggregationDto,
val deleted: Boolean,
)
fun SeriesDto.restrictUrl(restrict: Boolean) =

121
komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycleTest.kt
View file

@ -86,7 +86,7 @@ class LibraryContentLifecycleTest(
}
@Test
fun `given existing series when removing files and scanning then only updated Books are persisted`() {
fun `given existing series when removing files and scanning then updated Books are persisted and removed books are marked as such`() {
// given
val library = makeLibrary()
libraryRepository.insert(library)
@ -111,9 +111,41 @@ class LibraryContentLifecycleTest(
verify(exactly = 2) { mockScanner.scanRootFolder(any()) }
assertThat(allSeries).hasSize(1)
assertThat(allBooks).hasSize(1)
assertThat(allBooks.map { it.name }).containsExactly("book1")
assertThat(bookRepository.count()).describedAs("Orphan book has been removed").isEqualTo(1)
assertThat(allBooks).hasSize(2)
assertThat(allBooks.filter { it.deletedDate == null }.map { it.name }).containsExactly("book1")
assertThat(allBooks.filter { it.deletedDate != null }.map { it.name }).containsExactly("book2")
}
@Test
fun `given existing series when removing files and scanning, restoring files and scanning then restored books are available`() {
// given
val library = makeLibrary()
libraryRepository.insert(library)
val books = listOf(makeBook("book1"), makeBook("book2"))
val lessBooks = listOf(makeBook("book1"))
every { mockScanner.scanRootFolder(any()) }
.returnsMany(
mapOf(makeSeries(name = "series") to books).toScanResult(),
mapOf(makeSeries(name = "series") to lessBooks).toScanResult(),
mapOf(makeSeries(name = "series") to books).toScanResult(),
)
libraryContentLifecycle.scanRootFolder(library) // creation
libraryContentLifecycle.scanRootFolder(library) // deletion
// when
libraryContentLifecycle.scanRootFolder(library) // restore
// then
val allSeries = seriesRepository.findAll()
val allBooks = bookRepository.findAll().sortedBy { it.number }
verify(exactly = 3) { mockScanner.scanRootFolder(any()) }
assertThat(allSeries).hasSize(1)
assertThat(allBooks).hasSize(2)
assertThat(allBooks.map { it.deletedDate }).containsOnlyNulls()
}
@Test
@ -148,7 +180,7 @@ class LibraryContentLifecycleTest(
}
@Test
fun `given existing series when deleting all books and scanning then Series and Books are removed`() {
fun `given existing series when deleting all books and scanning then Series and Books are marked as deleted`() {
// given
val library = makeLibrary()
libraryRepository.insert(library)
@ -166,12 +198,49 @@ class LibraryContentLifecycleTest(
// then
verify(exactly = 2) { mockScanner.scanRootFolder(any()) }
assertThat(seriesRepository.count()).describedAs("Series repository should be empty").isEqualTo(0)
assertThat(bookRepository.count()).describedAs("Book repository should be empty").isEqualTo(0)
val allSeries = seriesRepository.findAll()
val allBooks = bookRepository.findAll()
assertThat(allSeries.map { it.deletedDate }).doesNotContainNull()
assertThat(allSeries).hasSize(1)
assertThat(allBooks.map { it.deletedDate }).doesNotContainNull()
assertThat(allBooks).hasSize(1)
}
@Test
fun `given existing Series when deleting all books of one series and scanning then series and its Books are removed`() {
fun `given existing series when deleting all books and scanning then restoring and scanning then Series and Books are available`() {
// given
val library = makeLibrary()
libraryRepository.insert(library)
val series = makeSeries(name = "series")
val book = makeBook("book1")
every { mockScanner.scanRootFolder(any()) }
.returnsMany(
mapOf(series to listOf(book)).toScanResult(),
emptyMap<Series, List<Book>>().toScanResult(),
mapOf(series to listOf(book)).toScanResult(),
)
libraryContentLifecycle.scanRootFolder(library) // creation
libraryContentLifecycle.scanRootFolder(library) // deletion
// when
libraryContentLifecycle.scanRootFolder(library) // restore
// then
verify(exactly = 3) { mockScanner.scanRootFolder(any()) }
val allSeries = seriesRepository.findAll()
val allBooks = bookRepository.findAll()
assertThat(allSeries.map { it.deletedDate }).containsOnlyNulls()
assertThat(allSeries).hasSize(1)
assertThat(allBooks.map { it.deletedDate }).containsOnlyNulls()
assertThat(allBooks).hasSize(1)
}
@Test
fun `given existing Series when deleting all books of one series and scanning then series and its Books are marked as deleted`() {
// given
val library = makeLibrary()
libraryRepository.insert(library)
@ -192,8 +261,20 @@ class LibraryContentLifecycleTest(
// then
verify(exactly = 2) { mockScanner.scanRootFolder(any()) }
assertThat(seriesRepository.count()).describedAs("Series repository should not be empty").isEqualTo(1)
assertThat(bookRepository.count()).describedAs("Book repository should not be empty").isEqualTo(1)
val (series, deletedSeries) = seriesRepository.findAll().partition { it.deletedDate == null }
val (books, deletedBooks) = bookRepository.findAll().partition { it.deletedDate == null }
assertThat(series).hasSize(1)
assertThat(series.map { it.name }).containsExactlyInAnyOrder("series")
assertThat(deletedSeries).hasSize(1)
assertThat(deletedSeries.map { it.name }).containsExactlyInAnyOrder("series2")
assertThat(books).hasSize(1)
assertThat(books.map { it.name }).containsExactlyInAnyOrder("book1")
assertThat(deletedBooks).hasSize(1)
assertThat(deletedBooks.map { it.name }).containsExactlyInAnyOrder("book2")
}
@Test
@ -294,8 +375,8 @@ class LibraryContentLifecycleTest(
libraryContentLifecycle.scanRootFolder(library1)
libraryContentLifecycle.scanRootFolder(library2)
assertThat(seriesRepository.count()).describedAs("Series repository should be empty").isEqualTo(2)
assertThat(bookRepository.count()).describedAs("Book repository should be empty").isEqualTo(2)
assertThat(seriesRepository.count()).describedAs("Series repository should not be empty").isEqualTo(2)
assertThat(bookRepository.count()).describedAs("Book repository should not be empty").isEqualTo(2)
// when
libraryContentLifecycle.scanRootFolder(library2)
@ -304,7 +385,19 @@ class LibraryContentLifecycleTest(
verify(exactly = 1) { mockScanner.scanRootFolder(Paths.get(library1.root.toURI())) }
verify(exactly = 2) { mockScanner.scanRootFolder(Paths.get(library2.root.toURI())) }
assertThat(seriesRepository.count()).describedAs("Series repository should be empty").isEqualTo(1)
assertThat(bookRepository.count()).describedAs("Book repository should be empty").isEqualTo(1)
val (seriesLib1, seriesLib2) = seriesRepository.findAll().partition { it.libraryId == library1.id }
val (booksLib1, booksLib2) = bookRepository.findAll().partition { it.libraryId == library1.id }
assertThat(seriesLib1.map { it.deletedDate }).containsOnlyNulls()
assertThat(seriesLib1.map { it.name }).containsExactlyInAnyOrder("series1")
assertThat(seriesLib2.map { it.deletedDate }).doesNotContainNull()
assertThat(seriesLib2.map { it.name }).containsExactlyInAnyOrder("series2")
assertThat(booksLib1.map { it.deletedDate }).containsOnlyNulls()
assertThat(booksLib1.map { it.name }).containsExactlyInAnyOrder("book1")
assertThat(booksLib2.map { it.deletedDate }).doesNotContainNull()
assertThat(booksLib2.map { it.name }).containsExactlyInAnyOrder("book2")
}
}

8
komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/BookDaoTest.kt
View file

@ -58,7 +58,8 @@ class BookDaoTest(
fileSize = 3,
fileHash = "abc",
seriesId = series.id,
libraryId = library.id
libraryId = library.id,
deletedDate = LocalDateTime.now(),
)
bookDao.insert(book)
@ -72,6 +73,7 @@ class BookDaoTest(
assertThat(created.fileLastModified).isEqualToIgnoringNanos(book.fileLastModified)
assertThat(created.fileSize).isEqualTo(book.fileSize)
assertThat(created.fileHash).isEqualTo(book.fileHash)
assertThat(created.deletedDate).isEqualTo(book.deletedDate)
}
@Test
@ -82,7 +84,7 @@ class BookDaoTest(
fileLastModified = LocalDateTime.now(),
fileSize = 3,
seriesId = series.id,
libraryId = library.id
libraryId = library.id,
)
bookDao.insert(book)
@ -95,6 +97,7 @@ class BookDaoTest(
fileLastModified = modificationDate,
fileSize = 5,
fileHash = "def",
deletedDate = LocalDateTime.now(),
)
}
@ -111,6 +114,7 @@ class BookDaoTest(
assertThat(modified.fileLastModified).isEqualToIgnoringNanos(modificationDate)
assertThat(modified.fileSize).isEqualTo(5)
assertThat(modified.fileHash).isEqualTo("def")
assertThat(modified.deletedDate).isEqualToIgnoringNanos(updated.deletedDate)
}
@Test

41
komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDaoTest.kt
View file

@ -48,7 +48,8 @@ class SeriesDaoTest(
name = "Series",
url = URL("file://series"),
fileLastModified = now,
libraryId = library.id
libraryId = library.id,
deletedDate = now,
)
seriesDao.insert(series)
@ -60,6 +61,44 @@ class SeriesDaoTest(
assertThat(created.name).isEqualTo(series.name)
assertThat(created.url).isEqualTo(series.url)
assertThat(created.fileLastModified).isEqualToIgnoringNanos(series.fileLastModified)
assertThat(created.deletedDate).isEqualToIgnoringNanos(series.deletedDate)
}
@Test
fun `given a series when updating then it is persisted`() {
val now = LocalDateTime.now()
val series = Series(
name = "Series",
url = URL("file://series"),
fileLastModified = now,
libraryId = library.id,
)
seriesDao.insert(series)
val modificationDate = LocalDateTime.now()
val updated = seriesDao.findByIdOrNull(series.id)!!.copy(
name = "Updated",
url = URL("file://updated"),
fileLastModified = modificationDate,
bookCount = 5,
deletedDate = LocalDateTime.now(),
)
seriesDao.update(updated)
val modified = seriesDao.findByIdOrNull(updated.id)!!
assertThat(modified.id).isEqualTo(updated.id)
assertThat(modified.createdDate).isEqualTo(updated.createdDate)
assertThat(modified.lastModifiedDate)
.isCloseTo(modificationDate, offset)
.isNotEqualTo(updated.lastModifiedDate)
assertThat(modified.name).isEqualTo("Updated")
assertThat(modified.url).isEqualTo(URL("file://updated"))
assertThat(modified.fileLastModified).isEqualToIgnoringNanos(modificationDate)
assertThat(modified.bookCount).isEqualTo(5)
assertThat(modified.deletedDate).isEqualToIgnoringNanos(updated.deletedDate)
}
@Test