From a3c3a48038ecde6a1be5c4795047f8a37fbb6e11 Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Mon, 16 Aug 2021 16:08:18 +0800 Subject: [PATCH] feat(importer): import sidecars alongside books closes #611 --- .../komga/domain/service/BookImporter.kt | 50 +++++++++++ .../komga/domain/service/FileSystemScanner.kt | 13 +++ .../komga/domain/service/BookImporterTest.kt | 84 ++++++++++++++++++- 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/BookImporter.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/BookImporter.kt index a464a06e..b4bcabe9 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/BookImporter.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/BookImporter.kt @@ -2,6 +2,7 @@ package org.gotson.komga.domain.service import mu.KotlinLogging import org.gotson.komga.application.events.EventPublisher +import org.gotson.komga.application.tasks.TaskReceiver import org.gotson.komga.domain.model.Book import org.gotson.komga.domain.model.CodedException import org.gotson.komga.domain.model.CopyMode @@ -9,6 +10,7 @@ import org.gotson.komga.domain.model.DomainEvent import org.gotson.komga.domain.model.Media import org.gotson.komga.domain.model.PathContainedInPath import org.gotson.komga.domain.model.Series +import org.gotson.komga.domain.model.Sidecar import org.gotson.komga.domain.model.withCode import org.gotson.komga.domain.persistence.BookMetadataRepository import org.gotson.komga.domain.persistence.BookRepository @@ -16,6 +18,7 @@ import org.gotson.komga.domain.persistence.LibraryRepository import org.gotson.komga.domain.persistence.MediaRepository import org.gotson.komga.domain.persistence.ReadListRepository import org.gotson.komga.domain.persistence.ReadProgressRepository +import org.gotson.komga.domain.persistence.SidecarRepository import org.gotson.komga.infrastructure.language.toIndexedMap import org.springframework.stereotype.Service import java.io.FileNotFoundException @@ -24,6 +27,7 @@ import java.nio.file.Files import java.nio.file.NoSuchFileException import java.nio.file.Path import java.nio.file.Paths +import java.nio.file.attribute.BasicFileAttributes import kotlin.io.path.copyTo import kotlin.io.path.deleteExisting import kotlin.io.path.deleteIfExists @@ -31,7 +35,10 @@ import kotlin.io.path.exists import kotlin.io.path.extension import kotlin.io.path.moveTo import kotlin.io.path.name +import kotlin.io.path.nameWithoutExtension import kotlin.io.path.notExists +import kotlin.io.path.readAttributes +import kotlin.io.path.toPath private val logger = KotlinLogging.logger {} @@ -46,7 +53,9 @@ class BookImporter( private val readProgressRepository: ReadProgressRepository, private val readListRepository: ReadListRepository, private val libraryRepository: LibraryRepository, + private val sidecarRepository: SidecarRepository, private val eventPublisher: EventPublisher, + private val taskReceiver: TaskReceiver, ) { fun importBook(sourceFile: Path, series: Series, copyMode: CopyMode, destinationName: String? = null, upgradeBookId: String? = null): Book { @@ -61,6 +70,12 @@ class BookImporter( if (destinationName != null) Paths.get("$destinationName.${sourceFile.extension}").name else sourceFile.name ) + val sidecars = fileSystemScanner.scanBookSidecars(sourceFile).associateWith { + series.path.resolve( + if (destinationName != null) it.url.toURI().toPath().name.replace(sourceFile.nameWithoutExtension, destinationName, true) + else it.url.toURI().toPath().name + ) + } val upgradedBook = if (upgradeBookId != null) { @@ -92,17 +107,39 @@ class BookImporter( CopyMode.MOVE -> { logger.info { "Moving file $sourceFile to $destFile" } sourceFile.moveTo(destFile) + sidecars.forEach { + it.key.url.toURI().toPath().let { sourcePath -> + logger.info { "Moving file $sourcePath to ${it.value}" } + sourcePath.moveTo(it.value, true) + } + } } CopyMode.COPY -> { logger.info { "Copying file $sourceFile to $destFile" } sourceFile.copyTo(destFile) + sidecars.forEach { + it.key.url.toURI().toPath().let { sourcePath -> + logger.info { "Copying file $sourcePath to ${it.value}" } + sourcePath.copyTo(it.value, true) + } + } } CopyMode.HARDLINK -> try { logger.info { "Hardlink file $sourceFile to $destFile" } Files.createLink(destFile, sourceFile) + sidecars.forEach { + it.key.url.toURI().toPath().let { sourcePath -> + logger.info { "Hardlink file $sourcePath to ${it.value}" } + it.value.deleteIfExists() + Files.createLink(it.value, sourcePath) + } + } } catch (e: Exception) { logger.warn(e) { "Filesystem does not support hardlinks, copying instead" } sourceFile.copyTo(destFile) + sidecars.forEach { + it.key.url.toURI().toPath().copyTo(it.value, true) + } } } @@ -153,6 +190,19 @@ class BookImporter( seriesLifecycle.sortBooks(series) + sidecars.forEach { (sourceSidecar, destPath) -> + when (sourceSidecar.type) { + Sidecar.Type.ARTWORK -> taskReceiver.refreshBookLocalArtwork(importedBook.id) + Sidecar.Type.METADATA -> taskReceiver.refreshBookMetadata(importedBook.id) + } + val destSidecar = sourceSidecar.copy( + url = destPath.toUri().toURL(), + parentUrl = destPath.parent.toUri().toURL(), + lastModifiedTime = destPath.readAttributes().getUpdatedTime() + ) + sidecarRepository.save(importedBook.libraryId, destSidecar) + } + eventPublisher.publishEvent(DomainEvent.BookImported(importedBook, sourceFile.toUri().toURL(), success = true)) return importedBook diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt index 8807ddfe..88e5acc1 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt @@ -23,6 +23,7 @@ import java.time.LocalDateTime import java.time.ZoneId import kotlin.io.path.exists import kotlin.io.path.extension +import kotlin.io.path.listDirectoryEntries import kotlin.io.path.name import kotlin.io.path.nameWithoutExtension import kotlin.io.path.pathString @@ -174,6 +175,18 @@ class FileSystemScanner( return pathToBook(path, path.readAttributes()) } + fun scanBookSidecars(path: Path): List { + val bookBaseName = path.nameWithoutExtension + val parent = path.parent + return parent.listDirectoryEntries() + .filter { candidate -> sidecarBookPrefilter.any { it.matches(candidate.name) } } + .mapNotNull { candidate -> + sidecarBookConsumers.firstOrNull { it.isSidecarBookMatch(bookBaseName, candidate.name) }?.let { + Sidecar(candidate.toUri().toURL(), parent.toUri().toURL(), candidate.readAttributes().getUpdatedTime(), it.getSidecarBookType(), Sidecar.Source.BOOK) + } + } + } + private fun pathToBook(path: Path, attrs: BasicFileAttributes): Book = Book( name = path.nameWithoutExtension, diff --git a/komga/src/test/kotlin/org/gotson/komga/domain/service/BookImporterTest.kt b/komga/src/test/kotlin/org/gotson/komga/domain/service/BookImporterTest.kt index 410cf9bc..44df97a9 100644 --- a/komga/src/test/kotlin/org/gotson/komga/domain/service/BookImporterTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/domain/service/BookImporterTest.kt @@ -2,8 +2,14 @@ package org.gotson.komga.domain.service import com.google.common.jimfs.Configuration import com.google.common.jimfs.Jimfs +import com.ninjasquad.springmockk.MockkBean +import io.mockk.Runs +import io.mockk.every +import io.mockk.just +import io.mockk.verify import org.assertj.core.api.Assertions import org.assertj.core.api.Assertions.assertThat +import org.gotson.komga.application.tasks.TaskReceiver import org.gotson.komga.domain.model.BookPage import org.gotson.komga.domain.model.CopyMode import org.gotson.komga.domain.model.KomgaUser @@ -25,6 +31,7 @@ import org.gotson.komga.infrastructure.language.toIndexedMap import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import org.springframework.beans.factory.annotation.Autowired @@ -59,6 +66,9 @@ class BookImporterTest( private val user1 = KomgaUser("user1@example.org", "", false) private val user2 = KomgaUser("user2@example.org", "", false) + @MockkBean + private lateinit var mockTackReceiver: TaskReceiver + @BeforeAll fun init() { libraryRepository.insert(library) @@ -74,6 +84,12 @@ class BookImporterTest( userRepository.deleteAll() } + @BeforeEach + fun initMocks() { + every { mockTackReceiver.refreshBookMetadata(any(), any(), any()) } just Runs + every { mockTackReceiver.refreshBookLocalArtwork(any(), any()) } just Runs + } + @AfterEach fun `clear repository`() { seriesLifecycle.deleteMany(seriesRepository.findAll()) @@ -136,7 +152,7 @@ class BookImporterTest( } @Test - fun `given source file parf of a Komga library when importing then exception is thrown`() { + fun `given source file part of a Komga library when importing then exception is thrown`() { Jimfs.newFileSystem(Configuration.unix()).use { fs -> // given val sourceDir = fs.getPath("/source").createDirectory() @@ -201,6 +217,72 @@ class BookImporterTest( } } + @Test + fun `given book with sidecars when importing then book and sidecars are imported`() { + Jimfs.newFileSystem(Configuration.unix()).use { fs -> + // given + val sourceDir = fs.getPath("/source").createDirectory() + val sourceFile = sourceDir.resolve("book 2.cbz").createFile() + sourceDir.resolve("book 2.jpg").createFile() + sourceDir.resolve("BOOK 2-1.jpg").createFile() + val destDir = fs.getPath("/library/series").createDirectories() + + val existingBooks = listOf( + makeBook("1", libraryId = library.id), + makeBook("3", libraryId = library.id), + ) + val series = makeSeries("series", url = destDir.toUri().toURL(), libraryId = library.id) + .also { series -> + seriesLifecycle.createSeries(series) + seriesLifecycle.addBooks(series, existingBooks) + seriesLifecycle.sortBooks(series) + } + + // when + bookImporter.importBook(sourceFile, series, CopyMode.COPY) + + // then + verify(exactly = 2) { mockTackReceiver.refreshBookLocalArtwork(any()) } + + assertThat(destDir.resolve("book 2.cbz")).exists() + assertThat(destDir.resolve("book 2.jpg")).exists() + assertThat(destDir.resolve("BOOK 2-1.jpg")).exists() + } + } + + @Test + fun `given book with sidecars when importing with destination name then book and sidecars are imported`() { + Jimfs.newFileSystem(Configuration.unix()).use { fs -> + // given + val sourceDir = fs.getPath("/source").createDirectory() + val sourceFile = sourceDir.resolve("book 2.cbz").createFile() + sourceDir.resolve("book 2.jpg").createFile() + sourceDir.resolve("BOOK 2-1.jpg").createFile() + val destDir = fs.getPath("/library/series").createDirectories() + + val existingBooks = listOf( + makeBook("1", libraryId = library.id), + makeBook("3", libraryId = library.id), + ) + val series = makeSeries("series", url = destDir.toUri().toURL(), libraryId = library.id) + .also { series -> + seriesLifecycle.createSeries(series) + seriesLifecycle.addBooks(series, existingBooks) + seriesLifecycle.sortBooks(series) + } + + // when + bookImporter.importBook(sourceFile, series, CopyMode.COPY, destinationName = "book 5") + + // then + verify(exactly = 2) { mockTackReceiver.refreshBookLocalArtwork(any()) } + + assertThat(destDir.resolve("book 5.cbz")).exists() + assertThat(destDir.resolve("book 5.jpg")).exists() + assertThat(destDir.resolve("book 5-1.jpg")).exists() + } + } + @Test fun `given existing book when importing with upgrade then existing book is deleted`() { Jimfs.newFileSystem(Configuration.unix()).use { fs ->