From 52168815d478b6edb961b895a751321d2823e456 Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Fri, 16 Aug 2019 17:05:26 +0800 Subject: [PATCH] added content detection for archives and archive content zip archives support reviewed JPA entities added logs --- komga/build.gradle.kts | 3 + .../org/gotson/komga/domain/model/Book.kt | 25 ++++-- .../gotson/komga/domain/model/BookMetadata.kt | 43 +++++++++++ .../org/gotson/komga/domain/model/BookPage.kt | 6 ++ .../org/gotson/komga/domain/model/Serie.kt | 19 +++-- .../persistence/BookMetadataRepository.kt | 8 ++ .../domain/persistence/SerieRepository.kt | 1 + .../komga/domain/service/BookManager.kt | 73 ++++++++++++++++++ .../gotson/komga/domain/service/BookParser.kt | 57 ++++++++++++++ .../komga/domain/service/FileSystemScanner.kt | 62 ++++++++------- .../komga/domain/service/LibraryManager.kt | 22 ++++-- .../infrastructure/archive/ContentDetector.kt | 39 ++++++++++ .../archive/TikaConfiguration.kt | 12 +++ .../infrastructure/archive/ZipExtractor.kt | 18 +++++ .../scheduler/RootScannerController.kt | 1 + .../komga/interfaces/web/SerieController.kt | 76 ++++++++++++++++++- .../org/gotson/komga/domain/model/Utils.kt | 10 +++ .../domain/persistence/PersistenceTest.kt | 70 +++++++++++++++++ .../domain/service/LibraryManagerTest.kt | 62 ++++++++++----- .../gotson/komga/infrastructure/ParserTest.kt | 28 +++++++ komga/src/test/resources/application-test.yml | 7 ++ 21 files changed, 577 insertions(+), 65 deletions(-) create mode 100644 komga/src/main/kotlin/org/gotson/komga/domain/model/BookMetadata.kt create mode 100644 komga/src/main/kotlin/org/gotson/komga/domain/model/BookPage.kt create mode 100644 komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookMetadataRepository.kt create mode 100644 komga/src/main/kotlin/org/gotson/komga/domain/service/BookManager.kt create mode 100644 komga/src/main/kotlin/org/gotson/komga/domain/service/BookParser.kt create mode 100644 komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/ContentDetector.kt create mode 100644 komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/TikaConfiguration.kt create mode 100644 komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/ZipExtractor.kt create mode 100644 komga/src/test/kotlin/org/gotson/komga/domain/model/Utils.kt create mode 100644 komga/src/test/kotlin/org/gotson/komga/domain/persistence/PersistenceTest.kt create mode 100644 komga/src/test/kotlin/org/gotson/komga/infrastructure/ParserTest.kt diff --git a/komga/build.gradle.kts b/komga/build.gradle.kts index 89fea899e..815503c1e 100644 --- a/komga/build.gradle.kts +++ b/komga/build.gradle.kts @@ -54,6 +54,9 @@ dependencies { implementation("commons-io:commons-io:2.6") + implementation("org.apache.tika:tika-core:1.22") + implementation("net.lingala.zip4j:zip4j:2.1.2") + runtimeOnly("com.h2database:h2:1.4.199") testImplementation("org.springframework.boot:spring-boot-starter-test") { diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/Book.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/Book.kt index 641cb33b4..4732ae0a0 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/Book.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/Book.kt @@ -1,28 +1,41 @@ package org.gotson.komga.domain.model import java.net.URL +import java.nio.file.Path +import java.nio.file.Paths import java.time.LocalDateTime +import javax.persistence.CascadeType import javax.persistence.Entity import javax.persistence.FetchType import javax.persistence.GeneratedValue import javax.persistence.Id import javax.persistence.ManyToOne +import javax.persistence.OneToOne import javax.validation.constraints.NotBlank import javax.validation.constraints.NotNull @Entity -data class Book( - @Id - @GeneratedValue - var id: Long? = null, - +class Book( @NotBlank val name: String, val url: URL, val updated: LocalDateTime ) { + @Id + @GeneratedValue + var id: Long = 0 + @NotNull @ManyToOne(fetch = FetchType.LAZY, optional = false) lateinit var serie: Serie -} \ No newline at end of file + + @OneToOne(optional = false, orphanRemoval = true, cascade = [CascadeType.ALL], fetch = FetchType.LAZY, mappedBy = "book") + var metadata: BookMetadata = BookMetadata().also { it.book = this } + set(value) { + value.book = this + field = value + } +} + +fun Book.path(): Path = Paths.get(this.url.toURI()) \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/BookMetadata.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/BookMetadata.kt new file mode 100644 index 000000000..70672684e --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/BookMetadata.kt @@ -0,0 +1,43 @@ +package org.gotson.komga.domain.model + +import javax.persistence.CollectionTable +import javax.persistence.Column +import javax.persistence.ElementCollection +import javax.persistence.Entity +import javax.persistence.EnumType +import javax.persistence.Enumerated +import javax.persistence.FetchType +import javax.persistence.GeneratedValue +import javax.persistence.Id +import javax.persistence.OneToOne + +@Entity +class BookMetadata( + @Enumerated(EnumType.STRING) + val status: Status = Status.UNKNOWN, + val mediaType: String? = null, + pages: List = emptyList() +) { + @Id + @GeneratedValue + val id: Long = 0 + + @OneToOne(optional = false, fetch = FetchType.LAZY) + lateinit var book: Book + + @ElementCollection(fetch = FetchType.EAGER) + @CollectionTable(name = "book_metadata_pages") + @Column(name = "pages") + private val _pages: MutableList = mutableListOf() + + val pages: List + get() = _pages.toList() + + init { + _pages.addAll(pages) + } +} + +enum class Status { + UNKNOWN, ERROR, READY, UNSUPPORTED +} \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/BookPage.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/BookPage.kt new file mode 100644 index 000000000..43b3b6563 --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/BookPage.kt @@ -0,0 +1,6 @@ +package org.gotson.komga.domain.model + +data class BookPage( + val mediaType: String, + val content: ByteArray +) \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/Serie.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/Serie.kt index d2ca5cb83..40c98939b 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/Serie.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/Serie.kt @@ -11,21 +11,28 @@ import javax.persistence.OneToMany import javax.validation.constraints.NotBlank @Entity -data class Serie( - @Id - @GeneratedValue - var id: Long? = null, - +class Serie( @NotBlank val name: String, val url: URL, val updated: LocalDateTime ) { + @Id + @GeneratedValue + var id: Long = 0 + @OneToMany(cascade = [CascadeType.ALL], fetch = FetchType.LAZY, mappedBy = "serie", orphanRemoval = true) - var books: MutableList = mutableListOf() + private var _books: MutableList = mutableListOf() set(value) { value.forEach { it.serie = this } field = value } + + val books: List + get() = _books.toList() + + fun setBooks(books: List) { + _books = books.toMutableList() + } } \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookMetadataRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookMetadataRepository.kt new file mode 100644 index 000000000..450d6831f --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookMetadataRepository.kt @@ -0,0 +1,8 @@ +package org.gotson.komga.domain.persistence + +import org.gotson.komga.domain.model.BookMetadata +import org.springframework.data.jpa.repository.JpaRepository +import org.springframework.stereotype.Repository + +@Repository +interface BookMetadataRepository : JpaRepository \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SerieRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SerieRepository.kt index d272964e2..164e6f4f0 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SerieRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SerieRepository.kt @@ -8,5 +8,6 @@ import java.net.URL @Repository interface SerieRepository : JpaRepository { fun deleteAllByUrlNotIn(urls: Iterable) + fun countByUrlNotIn(urls: Iterable): Long fun findByUrl(url: URL): Serie? } \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/BookManager.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/BookManager.kt new file mode 100644 index 000000000..ae6c36cac --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/BookManager.kt @@ -0,0 +1,73 @@ +package org.gotson.komga.domain.service + +import mu.KotlinLogging +import org.gotson.komga.domain.model.Book +import org.gotson.komga.domain.model.BookMetadata +import org.gotson.komga.domain.model.BookPage +import org.gotson.komga.domain.model.Status +import org.gotson.komga.domain.persistence.BookRepository +import org.gotson.komga.infrastructure.archive.ContentDetector +import org.springframework.stereotype.Service +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream + +private val logger = KotlinLogging.logger {} + +@Service +class BookManager( + private val bookRepository: BookRepository, + private val bookParser: BookParser, + private val contentDetector: ContentDetector +) { + + fun parseAndPersist(book: Book) { + logger.info { "Parse and persist book: ${book.url}" } + try { + book.metadata = bookParser.parse(book) + } catch (ex: UnsupportedMediaTypeException) { + logger.info(ex) { "Unsupported media type: ${ex.mediaType}" } + book.metadata = BookMetadata(status = Status.UNSUPPORTED, mediaType = ex.mediaType) + } catch (ex: Exception) { + logger.error(ex) { "Error while parsing" } + book.metadata = BookMetadata(status = Status.ERROR) + } + bookRepository.save(book) + } + + fun getPage(book: Book, number: Int): BookPage { + logger.info { "Get page #$number for book: ${book.url}" } + + if (book.metadata.status == Status.UNKNOWN) { + logger.info { "Book metadata is unknown, parsing it now" } + parseAndPersist(book) + } + + if (book.metadata.status != Status.READY) { + logger.warn { "Book metadata is not ready, cannot get pages" } + throw MetadataNotReadyException() + } + + lateinit var mediaType: String + lateinit var content: ByteArray + + bookParser.getPage(book, number).use { stream -> + if (stream.markSupported()) { + logger.debug { "Stream supports mark, passing it as is for content detection" } + mediaType = contentDetector.detectMediaType(stream) + content = stream.readBytes() + } else { + logger.debug { "Stream does not support mark, using a cloned stream for content detection" } + val buffer = ByteArrayOutputStream() + stream.copyTo(buffer) + val clonedStream = ByteArrayInputStream(buffer.toByteArray()) + + mediaType = clonedStream.use { contentDetector.detectMediaType(it) } + content = buffer.toByteArray() + } + } + + logger.info { "Page media type: $mediaType" } + + return BookPage(mediaType, content) + } +} \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/BookParser.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/BookParser.kt new file mode 100644 index 000000000..3fc1c7602 --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/BookParser.kt @@ -0,0 +1,57 @@ +package org.gotson.komga.domain.service + +import mu.KotlinLogging +import org.gotson.komga.domain.model.Book +import org.gotson.komga.domain.model.BookMetadata +import org.gotson.komga.domain.model.Status +import org.gotson.komga.domain.model.path +import org.gotson.komga.infrastructure.archive.ContentDetector +import org.gotson.komga.infrastructure.archive.ZipExtractor +import org.springframework.stereotype.Service +import java.io.InputStream + +private val logger = KotlinLogging.logger {} + +@Service +class BookParser( + private val contentDetector: ContentDetector, + private val zipExtractor: ZipExtractor +) { + + val supportedMimeTypes = listOf( + "application/zip" + ) + + fun parse(book: Book): BookMetadata { + logger.info { "Trying to parse book: ${book.url}" } + + val mediaType = contentDetector.detectMediaType(book.path()) + logger.info { "Detected media type: $mediaType" } + if (!supportedMimeTypes.contains(mediaType)) + throw UnsupportedMediaTypeException("Unsupported mime type: $mediaType. File: ${book.url}", mediaType) + + val pageNames = zipExtractor.getFilenames(book.path()) + logger.info { "Book has ${pageNames.size} pages" } + + return BookMetadata(mediaType = mediaType, status = Status.READY, pages = pageNames) + } + + fun getPage(book: Book, number: Int): InputStream { + logger.info { "Get page #$number for book: ${book.url}" } + + if (book.metadata.status != Status.READY) { + logger.warn { "Book metadata is not ready, cannot get pages" } + throw MetadataNotReadyException() + } + + if (number > book.metadata.pages.size || number <= 0) { + logger.error { "Page number #$number is out of bounds. Book has ${book.metadata.pages.size} pages" } + throw ArrayIndexOutOfBoundsException("Page $number does not exist") + } + + return zipExtractor.getEntryStream(book.path(), book.metadata.pages[number - 1]) + } +} + +class MetadataNotReadyException : Exception() +class UnsupportedMediaTypeException(msg: String, val mediaType: String) : Exception(msg) \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt index 1a46beae8..507dd7322 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt @@ -13,46 +13,56 @@ import java.time.LocalDateTime import java.time.ZoneId import kotlin.streams.asSequence import kotlin.streams.toList +import kotlin.system.measureTimeMillis private val logger = KotlinLogging.logger {} @Service -class FileSystemScanner( -) { +class FileSystemScanner { - val supportedExtensions = listOf("cbr", "rar", "cbz", "zip") + val supportedExtensions = listOf("cbz", "zip") fun scanRootFolder(root: Path): List { logger.info { "Scanning folder: $root" } + logger.info { "Supported extensions: $supportedExtensions" } - return Files.walk(root).asSequence() - .filter { !Files.isHidden(it) } - .filter { Files.isDirectory(it) } - .mapNotNull { dir -> - val books = Files.list(dir) - .filter { Files.isRegularFile(it) } - .filter { supportedExtensions.contains(FilenameUtils.getExtension(it.fileName.toString())) } - .map { - Book( - name = FilenameUtils.getBaseName(it.fileName.toString()), - url = it.toUri().toURL(), - updated = it.getUpdatedTime() - ) - }.toList() - if (books.isNullOrEmpty()) return@mapNotNull null - Serie( - name = dir.fileName.toString(), - url = dir.toUri().toURL(), - updated = dir.getUpdatedTime() - ).also { it.books = books.toMutableList() } - }.toList() + lateinit var scannedSeries: List + + measureTimeMillis { + scannedSeries = Files.walk(root).asSequence() + .filter { !Files.isHidden(it) } + .filter { Files.isDirectory(it) } + .mapNotNull { dir -> + val books = Files.list(dir) + .filter { Files.isRegularFile(it) } + .filter { supportedExtensions.contains(FilenameUtils.getExtension(it.fileName.toString())) } + .map { + Book( + name = FilenameUtils.getBaseName(it.fileName.toString()), + url = it.toUri().toURL(), + updated = it.getUpdatedTime() + ) + }.toList() + if (books.isNullOrEmpty()) return@mapNotNull null + Serie( + name = dir.fileName.toString(), + url = dir.toUri().toURL(), + updated = dir.getUpdatedTime() + ).also { it.setBooks(books) } + }.toList() + }.also { + val countOfBooks = scannedSeries.sumBy { it.books.size } + logger.info { "Scanned ${scannedSeries.size} series and $countOfBooks books in $it ms" } + } + + return scannedSeries } } -fun Path.getUpdatedTime() = +fun Path.getUpdatedTime(): LocalDateTime = Files.readAttributes(this, BasicFileAttributes::class.java).let { maxOf(it.creationTime(), it.lastModifiedTime()).toLocalDateTime() } -fun FileTime.toLocalDateTime() = +fun FileTime.toLocalDateTime(): LocalDateTime = LocalDateTime.ofInstant(this.toInstant(), ZoneId.systemDefault()) \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryManager.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryManager.kt index c54ac3cc4..589c45f3c 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryManager.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryManager.kt @@ -19,15 +19,22 @@ class LibraryManager( @Transactional fun scanRootFolder(library: Library) { - logger.info { "Scanning ${library.name}'s root folder: ${library.root}" } + logger.info { "Updating library: ${library.name}, root folder: ${library.root}" } measureTimeMillis { val series = fileSystemScanner.scanRootFolder(library.fileSystem.getPath(library.root)) // delete series that don't exist anymore - if (series.isEmpty()) + if (series.isEmpty()) { + logger.info { "Scan returned no series, deleting all existing series" } serieRepository.deleteAll() - else - serieRepository.deleteAllByUrlNotIn(series.map { it.url }) + } else { + val urls = series.map { it.url } + val countOfSeriesToDelete = serieRepository.countByUrlNotIn(urls) + if (countOfSeriesToDelete > 0) { + logger.info { "Deleting $countOfSeriesToDelete series not on disk anymore" } + serieRepository.deleteAllByUrlNotIn(urls) + } + } // match IDs for existing entities series.forEach { newSerie -> @@ -36,6 +43,11 @@ class LibraryManager( newSerie.books.forEach { newBook -> bookRepository.findByUrl(newBook.url)?.let { existingBook -> newBook.id = existingBook.id + // conserve metadata if book has not changed + if (newBook.updated == existingBook.updated) + newBook.metadata = existingBook.metadata + else + logger.info { "Book changed on disk, reset metadata status: ${newBook.url}" } } } } @@ -43,6 +55,6 @@ class LibraryManager( serieRepository.saveAll(series) - }.also { logger.info { "Scan finished in $it ms" } } + }.also { logger.info { "Update finished in $it ms" } } } } \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/ContentDetector.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/ContentDetector.kt new file mode 100644 index 000000000..508a810de --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/ContentDetector.kt @@ -0,0 +1,39 @@ +package org.gotson.komga.infrastructure.archive + +import mu.KotlinLogging +import org.apache.tika.config.TikaConfig +import org.apache.tika.io.TikaInputStream +import org.apache.tika.metadata.Metadata +import org.springframework.stereotype.Service +import java.io.InputStream +import java.nio.file.Path + +private val logger = KotlinLogging.logger {} + +@Service +class ContentDetector( + private val tika: TikaConfig +) { + + fun detectMediaType(path: Path): String { + logger.info { "detect media type for path: $path" } + + val metadata = Metadata().also { + it[Metadata.RESOURCE_NAME_KEY] = path.fileName.toString() + } + val mediaType = tika.detector.detect(TikaInputStream.get(path), metadata) + + logger.info { "media type detected: $mediaType" } + + return mediaType.toString() + } + + fun detectMediaType(stream: InputStream): String { + logger.info { "detect media type for stream" } + stream.use { + val mediaType = tika.detector.detect(TikaInputStream.get(it), Metadata()) + logger.info { "media type detected: $mediaType" } + return mediaType.toString() + } + } +} \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/TikaConfiguration.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/TikaConfiguration.kt new file mode 100644 index 000000000..5a415000f --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/TikaConfiguration.kt @@ -0,0 +1,12 @@ +package org.gotson.komga.infrastructure.archive + +import org.apache.tika.config.TikaConfig +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +@Configuration +class TikaConfiguration { + + @Bean + fun tika() = TikaConfig() +} \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/ZipExtractor.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/ZipExtractor.kt new file mode 100644 index 000000000..10f89d9fd --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/archive/ZipExtractor.kt @@ -0,0 +1,18 @@ +package org.gotson.komga.infrastructure.archive + +import net.lingala.zip4j.ZipFile +import org.springframework.stereotype.Service +import java.io.InputStream +import java.nio.file.Path + +@Service +class ZipExtractor { + + fun getFilenames(path: Path) = + ZipFile(path.toFile()).fileHeaders.map { it.fileName }.toMutableList() + + fun getEntryStream(path: Path, entryName: String): InputStream = + ZipFile(path.toFile()).let { + it.getInputStream(it.getFileHeader(entryName)) + } +} \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/RootScannerController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/RootScannerController.kt index 36371f41d..640fe80d0 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/RootScannerController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/RootScannerController.kt @@ -22,6 +22,7 @@ class RootScannerController( @EventListener(ApplicationReadyEvent::class) @Scheduled(cron = "#{@komgaProperties.rootFolderScanCron ?: '-'}") fun scanRootFolder() { + logger.info { "Starting periodic library scan" } libraryManager.scanRootFolder(Library("default", komgaProperties.rootFolder)) } } \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/web/SerieController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/web/SerieController.kt index 4eac3a89c..7d777d859 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/web/SerieController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/web/SerieController.kt @@ -2,13 +2,17 @@ package org.gotson.komga.interfaces.web import org.gotson.komga.domain.model.Book import org.gotson.komga.domain.model.Serie +import org.gotson.komga.domain.model.Status import org.gotson.komga.domain.persistence.BookRepository import org.gotson.komga.domain.persistence.SerieRepository +import org.gotson.komga.domain.service.BookManager +import org.gotson.komga.domain.service.MetadataNotReadyException import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable import org.springframework.data.repository.findByIdOrNull import org.springframework.http.HttpStatus import org.springframework.http.MediaType +import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestMapping @@ -21,7 +25,8 @@ import java.net.URL @RequestMapping("api/v1/series") class SerieController( private val serieRepository: SerieRepository, - private val bookRepository: BookRepository + private val bookRepository: BookRepository, + private val bookManager: BookManager ) { @GetMapping @@ -62,6 +67,49 @@ class SerieController( File(it.url.toURI()).readBytes() } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) } + + @GetMapping("{serieId}/books/{bookId}/pages") + fun getBookPages( + @PathVariable serieId: Long, + @PathVariable bookId: Long + ): List { + if (!serieRepository.existsById(serieId)) throw ResponseStatusException(HttpStatus.NOT_FOUND) + return bookRepository.findByIdOrNull((bookId))?.let { + if (it.metadata.status == Status.UNKNOWN) bookManager.parseAndPersist(it) + if (it.metadata.status in listOf(Status.ERROR, Status.UNSUPPORTED)) throw ResponseStatusException(HttpStatus.NO_CONTENT, "Book cannot be parsed") + + it.metadata.pages.mapIndexed { index, s -> PageDto(index + 1, s) } + } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) + } + + @GetMapping("{serieId}/books/{bookId}/pages/{pageNumber}") + fun getBookPage( + @PathVariable serieId: Long, + @PathVariable bookId: Long, + @PathVariable pageNumber: Int + ): ResponseEntity { + if (!serieRepository.existsById(serieId)) throw ResponseStatusException(HttpStatus.NOT_FOUND) + + try { + return bookRepository.findByIdOrNull((bookId))?.let { book -> + val page = bookManager.getPage(book, pageNumber) + + val mediaType = try { + MediaType.parseMediaType(page.mediaType) + } catch (ex: Exception) { + MediaType.APPLICATION_OCTET_STREAM + } + + ResponseEntity.ok() + .contentType(mediaType) + .body(page.content) + } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) + } catch (ex: ArrayIndexOutOfBoundsException) { + throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Page number does not exist") + } catch (ex: MetadataNotReadyException) { + throw ResponseStatusException(HttpStatus.NO_CONTENT, "Book cannot be parsed") + } + } } data class SerieDto( @@ -70,13 +118,33 @@ data class SerieDto( val url: URL ) -fun Serie.toDto() = SerieDto(id!!, name, url) +fun Serie.toDto() = SerieDto(id = id, name = name, url = url) data class BookDto( val id: Long, val name: String, - val url: URL + val url: URL, + val metadata: BookMetadataDto ) -fun Book.toDto() = BookDto(id!!, name, url) \ No newline at end of file +data class BookMetadataDto( + val status: String, + val mediaType: String +) + +fun Book.toDto() = + BookDto( + id = id, + name = name, + url = url, + metadata = BookMetadataDto( + status = metadata.status.toString(), + mediaType = metadata.mediaType ?: "" + ) + ) + +data class PageDto( + val number: Int, + val fileName: String +) \ No newline at end of file diff --git a/komga/src/test/kotlin/org/gotson/komga/domain/model/Utils.kt b/komga/src/test/kotlin/org/gotson/komga/domain/model/Utils.kt new file mode 100644 index 000000000..5fd6f894e --- /dev/null +++ b/komga/src/test/kotlin/org/gotson/komga/domain/model/Utils.kt @@ -0,0 +1,10 @@ +package org.gotson.komga.domain.model + +import java.net.URL +import java.time.LocalDateTime + +fun makeBook(name: String, url: String = "file:/$name") = + Book(name = name, url = URL(url), updated = LocalDateTime.now()) + +fun makeSerie(name: String, url: String = "file:/$name", books: List = listOf()) = + Serie(name = name, url = URL(url), updated = LocalDateTime.now()).also { it.setBooks(books) } \ No newline at end of file diff --git a/komga/src/test/kotlin/org/gotson/komga/domain/persistence/PersistenceTest.kt b/komga/src/test/kotlin/org/gotson/komga/domain/persistence/PersistenceTest.kt new file mode 100644 index 000000000..1b6c4b8b7 --- /dev/null +++ b/komga/src/test/kotlin/org/gotson/komga/domain/persistence/PersistenceTest.kt @@ -0,0 +1,70 @@ +package org.gotson.komga.domain.persistence + +import org.assertj.core.api.Assertions.assertThat +import org.gotson.komga.domain.model.BookMetadata +import org.gotson.komga.domain.model.Status +import org.gotson.komga.domain.model.makeBook +import org.gotson.komga.domain.model.makeSerie +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest +import org.springframework.boot.test.autoconfigure.orm.jpa.TestEntityManager +import org.springframework.test.context.junit.jupiter.SpringExtension +import org.springframework.transaction.annotation.Transactional + +@ExtendWith(SpringExtension::class) +@DataJpaTest +@Transactional +class PersistenceTest( + @Autowired private val serieRepository: SerieRepository, + @Autowired private val bookRepository: BookRepository, + @Autowired private val bookMetadataRepository: BookMetadataRepository, + @Autowired private val entityManager: TestEntityManager +) { + + @AfterEach + fun `clear repository`() { + entityManager.clear() + } + + @Test + fun `given serie with book when saving then metadata is also saved`() { + // given + val serie = makeSerie(name = "serie", books = mutableListOf(makeBook("book1"))) + + // when + serieRepository.save(serie) + + // then + assertThat(serieRepository.count()).isEqualTo(1) + assertThat(bookRepository.count()).isEqualTo(1) + assertThat(bookMetadataRepository.count()).isEqualTo(1) + } + + @Test + fun `given existing book when updating metadata then new metadata is saved`() { + // given + val serie = makeSerie(name = "serie", books = mutableListOf(makeBook("book1"))) + serieRepository.save(serie) + + // when + val book = bookRepository.findAll().first() + book.metadata = BookMetadata(status = Status.READY, mediaType = "test", pages = listOf("page1")) + + bookRepository.save(book) + + // then + assertThat(serieRepository.count()).isEqualTo(1) + assertThat(bookRepository.count()).isEqualTo(1) + assertThat(bookMetadataRepository.count()).isEqualTo(1) + bookMetadataRepository.findAll().first().let { + assertThat(it.status == Status.READY) + assertThat(it.mediaType == "test") + assertThat(it.pages) + .hasSize(1) + .containsExactly("page1") + } + } +} \ No newline at end of file diff --git a/komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryManagerTest.kt b/komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryManagerTest.kt index b9db48886..06138e2c5 100644 --- a/komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryManagerTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryManagerTest.kt @@ -4,9 +4,11 @@ import com.ninjasquad.springmockk.MockkBean import io.mockk.every import io.mockk.verify import org.assertj.core.api.Assertions.assertThat -import org.gotson.komga.domain.model.Book +import org.gotson.komga.domain.model.BookMetadata import org.gotson.komga.domain.model.Library -import org.gotson.komga.domain.model.Serie +import org.gotson.komga.domain.model.Status +import org.gotson.komga.domain.model.makeBook +import org.gotson.komga.domain.model.makeSerie import org.gotson.komga.domain.persistence.BookRepository import org.gotson.komga.domain.persistence.SerieRepository import org.junit.jupiter.api.AfterEach @@ -17,8 +19,6 @@ import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabas import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.junit.jupiter.SpringExtension import org.springframework.transaction.annotation.Transactional -import java.net.URL -import java.time.LocalDateTime @ExtendWith(SpringExtension::class) @SpringBootTest @@ -27,12 +27,16 @@ import java.time.LocalDateTime class LibraryManagerTest( @Autowired private val serieRepository: SerieRepository, @Autowired private val bookRepository: BookRepository, - @Autowired private val libraryManager: LibraryManager + @Autowired private val libraryManager: LibraryManager, + @Autowired private val bookManager: BookManager ) { @MockkBean private lateinit var mockScanner: FileSystemScanner + @MockkBean + private lateinit var mockParser: BookParser + private val library = Library(name = "test", root = "/root") @AfterEach @@ -40,17 +44,11 @@ class LibraryManagerTest( serieRepository.deleteAll() } - private fun makeBook(name: String, url: String = "file:/$name") = - Book(name = name, url = URL(url), updated = LocalDateTime.now()) - - private fun makeSerie(name: String, url: String = "file:/$name", books: MutableList = mutableListOf()) = - Serie(name = name, url = URL(url), updated = LocalDateTime.now()).also { it.books = books } - @Test fun `given existing Serie when adding files and scanning then only updated Books are persisted`() { //given - val serie = makeSerie(name = "serie", books = mutableListOf(makeBook("book1"))) - val serieWithMoreBooks = makeSerie(name = "serie", books = mutableListOf(makeBook("book1"), makeBook("book2"))) + val serie = makeSerie(name = "serie", books = listOf(makeBook("book1"))) + val serieWithMoreBooks = makeSerie(name = "serie", books = listOf(makeBook("book1"), makeBook("book2"))) every { mockScanner.scanRootFolder(any()) }.returnsMany( listOf(serie), @@ -74,8 +72,8 @@ class LibraryManagerTest( @Test fun `given existing Serie when removing files and scanning then only updated Books are persisted`() { //given - val serie = makeSerie(name = "serie", books = mutableListOf(makeBook("book1"), makeBook("book2"))) - val serieWithLessBooks = makeSerie(name = "serie", books = mutableListOf(makeBook("book1"))) + val serie = makeSerie(name = "serie", books = listOf(makeBook("book1"), makeBook("book2"))) + val serieWithLessBooks = makeSerie(name = "serie", books = listOf(makeBook("book1"))) every { mockScanner.scanRootFolder(any()) } .returnsMany( @@ -101,8 +99,8 @@ class LibraryManagerTest( @Test fun `given existing Serie when updating files and scanning then Books are updated`() { //given - val serie = makeSerie(name = "serie", books = mutableListOf(makeBook("book1"))) - val serieWithUpdatedBooks = makeSerie(name = "serie", books = mutableListOf(makeBook("book1updated", "file:/book1"))) + val serie = makeSerie(name = "serie", books = listOf(makeBook("book1"))) + val serieWithUpdatedBooks = makeSerie(name = "serie", books = listOf(makeBook("book1updated", "file:/book1"))) every { mockScanner.scanRootFolder(any()) } .returnsMany( @@ -127,7 +125,7 @@ class LibraryManagerTest( @Test fun `given existing Serie when deleting all books and scanning then Series and Books are removed`() { //given - val serie = makeSerie(name = "serie", books = mutableListOf(makeBook("book1"))) + val serie = makeSerie(name = "serie", books = listOf(makeBook("book1"))) every { mockScanner.scanRootFolder(any()) } .returnsMany( @@ -145,4 +143,32 @@ class LibraryManagerTest( assertThat(serieRepository.count()).describedAs("Serie repository should be empty").isEqualTo(0) assertThat(bookRepository.count()).describedAs("Book repository should be empty").isEqualTo(0) } + + @Test + fun `given existing Book with metadata when rescanning then metadata is kept intact`() { + //given + every { mockScanner.scanRootFolder(any()) } + .returnsMany( + listOf(makeSerie(name = "serie", books = listOf(makeBook("book1")))), + listOf(makeSerie(name = "serie", books = listOf(makeBook("book1")))) + ) + libraryManager.scanRootFolder(library) + + every { mockParser.parse(any()) } returns BookMetadata(status = Status.READY, mediaType = "application/zip", pages = listOf("1.jpg", "2.jpg")) + bookRepository.findAll().forEach { bookManager.parseAndPersist(it) } + + // when + libraryManager.scanRootFolder(library) + + // then + verify(exactly = 2) { mockScanner.scanRootFolder(any()) } + verify(exactly = 1) { mockParser.parse(any()) } + + val book = bookRepository.findAll().first() + assertThat(book.metadata.status).isEqualTo(Status.READY) + assertThat(book.metadata.mediaType).isEqualTo("application/zip") + assertThat(book.metadata.pages) + .hasSize(2) + .containsExactly("1.jpg", "2.jpg") + } } \ No newline at end of file diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/ParserTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/ParserTest.kt new file mode 100644 index 000000000..8cdc613fd --- /dev/null +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/ParserTest.kt @@ -0,0 +1,28 @@ +package org.gotson.komga.infrastructure + +import mu.KotlinLogging +import org.apache.tika.config.TikaConfig +import org.apache.tika.io.TikaInputStream +import org.apache.tika.metadata.Metadata +import org.junit.jupiter.api.Test +import java.nio.file.Paths + +private val logger = KotlinLogging.logger {} + +class ParserTest { + + @Test + fun parseTest() { + val filePath = """D:\files\comics\Chrononauts\Chrononauts 001 (2015) (Digital) (Zone-Empire).cbz""" + val path = Paths.get(filePath) + + val tika = TikaConfig() + + val metadata = Metadata().also { + it[Metadata.RESOURCE_NAME_KEY] = path.fileName.toString() + } + val mimeType = tika.detector.detect(TikaInputStream.get(path), metadata) + logger.info { mimeType } + logger.info { tika.detector.detect(TikaInputStream.get(path), Metadata()) } + } +} \ No newline at end of file diff --git a/komga/src/test/resources/application-test.yml b/komga/src/test/resources/application-test.yml index 5946d9847..8c0a9c61b 100644 --- a/komga/src/test/resources/application-test.yml +++ b/komga/src/test/resources/application-test.yml @@ -1 +1,8 @@ application.version: TESTING + +spring: + jpa: + show-sql: true + properties: + hibernate: + format_sql: true