diff --git a/komga/src/flyway/resources/db/migration/sqlite/V20210504171645__library_convert_to_cbz.sql b/komga/src/flyway/resources/db/migration/sqlite/V20210504171645__library_convert_to_cbz.sql new file mode 100644 index 000000000..1b6e3c477 --- /dev/null +++ b/komga/src/flyway/resources/db/migration/sqlite/V20210504171645__library_convert_to_cbz.sql @@ -0,0 +1,2 @@ +alter table library + add column CONVERT_TO_CBZ boolean NOT NULL DEFAULT 0; diff --git a/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt b/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt index 0d2fabfb3..90982e03a 100644 --- a/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt +++ b/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt @@ -7,6 +7,7 @@ import java.io.Serializable const val HIGHEST_PRIORITY = 8 const val HIGH_PRIORITY = 6 const val DEFAULT_PRIORITY = 4 +const val LOWEST_PRIORITY = 0 sealed class Task(priority: Int = DEFAULT_PRIORITY) : Serializable { abstract fun uniqueId(): String @@ -44,4 +45,9 @@ sealed class Task(priority: Int = DEFAULT_PRIORITY) : Serializable { override fun toString(): String = "ImportBook(sourceFile='$sourceFile', seriesId='$seriesId', copyMode=$copyMode, destinationName=$destinationName, upgradeBookId=$upgradeBookId, priority='$priority')" } + + class ConvertBook(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { + override fun uniqueId(): String = "CONVERT_BOOK_$bookId" + override fun toString(): String = "ConvertBook(bookId='$bookId', priority='$priority')" + } } diff --git a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt index 23ee89a54..31de55bd5 100644 --- a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt +++ b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt @@ -4,6 +4,7 @@ import mu.KotlinLogging import org.gotson.komga.domain.persistence.BookRepository import org.gotson.komga.domain.persistence.LibraryRepository import org.gotson.komga.domain.persistence.SeriesRepository +import org.gotson.komga.domain.service.BookConverter import org.gotson.komga.domain.service.BookImporter import org.gotson.komga.domain.service.BookLifecycle import org.gotson.komga.domain.service.LibraryContentLifecycle @@ -27,6 +28,7 @@ class TaskHandler( private val bookLifecycle: BookLifecycle, private val metadataLifecycle: MetadataLifecycle, private val bookImporter: BookImporter, + private val bookConverter: BookConverter, ) { @JmsListener(destination = QUEUE_TASKS, selector = QUEUE_TASKS_SELECTOR) @@ -37,9 +39,10 @@ class TaskHandler( when (task) { is Task.ScanLibrary -> - libraryRepository.findByIdOrNull(task.libraryId)?.let { - libraryContentLifecycle.scanRootFolder(it) - taskReceiver.analyzeUnknownAndOutdatedBooks(it) + libraryRepository.findByIdOrNull(task.libraryId)?.let { library -> + libraryContentLifecycle.scanRootFolder(library) + taskReceiver.analyzeUnknownAndOutdatedBooks(library) + if (library.convertToCbz) taskReceiver.convertBooksToCbz(library, LOWEST_PRIORITY) } ?: logger.warn { "Cannot execute task $task: Library does not exist" } is Task.AnalyzeBook -> @@ -77,6 +80,11 @@ class TaskHandler( val importedBook = bookImporter.importBook(Paths.get(task.sourceFile), series, task.copyMode, task.destinationName, task.upgradeBookId) taskReceiver.analyzeBook(importedBook, priority = task.priority + 1) } ?: logger.warn { "Cannot execute task $task: Series does not exist" } + + is Task.ConvertBook -> + bookRepository.findByIdOrNull(task.bookId)?.let { book -> + bookConverter.convertToCbz(book) + } } }.also { logger.info { "Task $task executed in $it" } diff --git a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt index 817e3447e..26d7e2c23 100644 --- a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt +++ b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt @@ -9,6 +9,8 @@ import org.gotson.komga.domain.model.Library import org.gotson.komga.domain.model.Media import org.gotson.komga.domain.persistence.BookRepository import org.gotson.komga.domain.persistence.LibraryRepository +import org.gotson.komga.domain.persistence.MediaRepository +import org.gotson.komga.domain.service.BookConverter import org.gotson.komga.infrastructure.jms.QUEUE_TASKS import org.gotson.komga.infrastructure.jms.QUEUE_TASKS_TYPE import org.gotson.komga.infrastructure.jms.QUEUE_TYPE @@ -23,7 +25,9 @@ private val logger = KotlinLogging.logger {} class TaskReceiver( private val jmsTemplate: JmsTemplate, private val libraryRepository: LibraryRepository, - private val bookRepository: BookRepository + private val bookRepository: BookRepository, + private val mediaRepository: MediaRepository, + private val bookConverter: BookConverter, ) { fun scanLibraries() { @@ -46,6 +50,12 @@ class TaskReceiver( } } + fun convertBooksToCbz(library: Library, priority: Int = DEFAULT_PRIORITY) { + mediaRepository.findBookIdsByMediaType(bookConverter.convertibleTypes).forEach { + submitTask(Task.ConvertBook(it, priority)) + } + } + fun analyzeBook(bookId: String, priority: Int = DEFAULT_PRIORITY) { submitTask(Task.AnalyzeBook(bookId, priority)) } @@ -86,6 +96,10 @@ class TaskReceiver( submitTask(Task.ImportBook(sourceFile, seriesId, copyMode, destinationName, upgradeBookId, priority)) } + fun convertBook(bookId: String, priority: Int = DEFAULT_PRIORITY) { + submitTask(Task.ConvertBook(bookId, priority)) + } + private fun submitTask(task: Task) { logger.info { "Sending task: $task" } jmsTemplate.priority = task.priority diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/Exceptions.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/Exceptions.kt index df0deb1c3..7e3bae9c1 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/Exceptions.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/Exceptions.kt @@ -8,3 +8,4 @@ class DirectoryNotFoundException(message: String, code: String = "") : CodedExce class DuplicateNameException(message: String, code: String = "") : CodedException(message, code) class PathContainedInPath(message: String, code: String = "") : CodedException(message, code) class UserEmailAlreadyExistsException(message: String, code: String = "") : CodedException(message, code) +class BookConversionException(message: String) : Exception(message) diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt index 46af269c8..37be4ff80 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt @@ -19,6 +19,7 @@ data class Library( val importBarcodeIsbn: Boolean = true, val scanForceModifiedTime: Boolean = false, val scanDeep: Boolean = false, + val convertToCbz: Boolean = false, val id: String = TsidCreator.getTsid256().toString(), diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/MediaRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/MediaRepository.kt index 7f14f6da2..acb5f8a4e 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/MediaRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/MediaRepository.kt @@ -4,6 +4,7 @@ import org.gotson.komga.domain.model.Media interface MediaRepository { fun findById(bookId: String): Media + fun findBookIdsByMediaType(mediaTypes: Collection): Collection fun insert(media: Media) fun insertMany(medias: Collection) diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/BookConverter.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/BookConverter.kt new file mode 100644 index 000000000..8f7bf2b75 --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/BookConverter.kt @@ -0,0 +1,105 @@ +package org.gotson.komga.domain.service + +import mu.KotlinLogging +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream +import org.apache.tika.mime.MediaType +import org.gotson.komga.domain.model.Book +import org.gotson.komga.domain.model.BookConversionException +import org.gotson.komga.domain.model.BookWithMedia +import org.gotson.komga.domain.model.Media +import org.gotson.komga.domain.model.MediaNotReadyException +import org.gotson.komga.domain.model.MediaUnsupportedException +import org.gotson.komga.domain.persistence.BookRepository +import org.gotson.komga.domain.persistence.LibraryRepository +import org.gotson.komga.domain.persistence.MediaRepository +import org.springframework.stereotype.Service +import java.io.FileNotFoundException +import java.nio.file.FileAlreadyExistsException +import java.util.zip.Deflater +import kotlin.io.path.deleteIfExists +import kotlin.io.path.exists +import kotlin.io.path.nameWithoutExtension +import kotlin.io.path.notExists +import kotlin.io.path.outputStream + +private val logger = KotlinLogging.logger {} + +const val CBZ_EXTENSION = "cbz" + +@Service +class BookConverter( + private val bookAnalyzer: BookAnalyzer, + private val fileSystemScanner: FileSystemScanner, + private val bookRepository: BookRepository, + private val mediaRepository: MediaRepository, + private val libraryRepository: LibraryRepository, +) { + + val convertibleTypes = listOf("application/x-rar-compressed; version=4") + + private val exclude = mutableListOf() + + fun convertToCbz(book: Book) { + if (!libraryRepository.findById(book.libraryId).convertToCbz) + return logger.info { "Book conversion is disabled for the library, it may have changed since the task was submitted, skipping" } + + if (exclude.contains(book.id)) + return logger.info { "Book conversion already failed before, skipping" } + + if (book.path.notExists()) throw FileNotFoundException("File not found: ${book.path}") + + val media = mediaRepository.findById(book.id) + + if (!convertibleTypes.contains(media.mediaType)) + throw MediaUnsupportedException("${media.mediaType} cannot be converted. Must be one of $convertibleTypes") + if (media.status != Media.Status.READY) + throw MediaNotReadyException() + + val destinationFilename = "${book.path.nameWithoutExtension}.$CBZ_EXTENSION" + val destinationPath = book.path.parent.resolve(destinationFilename) + if (destinationPath.exists()) + throw FileAlreadyExistsException("Destination file already exists: $destinationPath") + + logger.info { "Copying archive content to $destinationPath" } + ZipArchiveOutputStream(destinationPath.outputStream()).use { zipStream -> + zipStream.setMethod(ZipArchiveOutputStream.DEFLATED) + zipStream.setLevel(Deflater.NO_COMPRESSION) + + media + .pages.map { it.fileName } + .union(media.files) + .forEach { entry -> + zipStream.putArchiveEntry(ZipArchiveEntry(entry)) + zipStream.write(bookAnalyzer.getFileContent(BookWithMedia(book, media), entry)) + zipStream.closeArchiveEntry() + } + } + + val convertedBook = fileSystemScanner.scanFile(destinationPath) + ?.copy( + id = book.id, + seriesId = book.seriesId, + libraryId = book.libraryId + ) + ?: throw IllegalStateException("Newly converted book could not be scanned: $destinationFilename") + + val convertedMedia = bookAnalyzer.analyze(convertedBook) + + if (convertedMedia.status != Media.Status.READY || + convertedMedia.mediaType != MediaType.APPLICATION_ZIP.toString() || + !convertedMedia.pages.containsAll(media.pages) || + !convertedMedia.files.containsAll(media.files) + ) { + destinationPath.deleteIfExists() + exclude += book.id + throw BookConversionException("Converted file does not match existing file, aborting conversion") + } + + if (book.path.deleteIfExists()) + logger.info { "Deleted converted file: ${book.path}" } + + bookRepository.update(convertedBook) + mediaRepository.update(convertedMedia) + } +} diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt index d2a30f639..a8588560f 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt @@ -75,6 +75,7 @@ class LibraryDao( .set(l.IMPORT_BARCODE_ISBN, library.importBarcodeIsbn) .set(l.SCAN_FORCE_MODIFIED_TIME, library.scanForceModifiedTime) .set(l.SCAN_DEEP, library.scanDeep) + .set(l.CONVERT_TO_CBZ, library.convertToCbz) .execute() } @@ -92,6 +93,7 @@ class LibraryDao( .set(l.IMPORT_BARCODE_ISBN, library.importBarcodeIsbn) .set(l.SCAN_FORCE_MODIFIED_TIME, library.scanForceModifiedTime) .set(l.SCAN_DEEP, library.scanDeep) + .set(l.CONVERT_TO_CBZ, library.convertToCbz) .set(l.LAST_MODIFIED_DATE, LocalDateTime.now(ZoneId.of("Z"))) .where(l.ID.eq(library.id)) .execute() @@ -113,6 +115,7 @@ class LibraryDao( importBarcodeIsbn = importBarcodeIsbn, scanForceModifiedTime = scanForceModifiedTime, scanDeep = scanDeep, + convertToCbz = convertToCbz, id = id, createdDate = createdDate.toCurrentTimeZone(), lastModifiedDate = lastModifiedDate.toCurrentTimeZone() diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt index 7a8ea7468..429085947 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt @@ -26,6 +26,12 @@ class MediaDao( override fun findById(bookId: String): Media = find(dsl, bookId) + override fun findBookIdsByMediaType(mediaTypes: Collection): Collection = + dsl.select(m.BOOK_ID) + .from(m) + .where(m.MEDIA_TYPE.`in`(mediaTypes)) + .fetch(m.BOOK_ID) + private fun find(dsl: DSLContext, bookId: String): Media = dsl.select(*groupFields) .from(m) diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/LibraryController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/LibraryController.kt index 48858cf20..9999484c8 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/LibraryController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/LibraryController.kt @@ -83,7 +83,8 @@ class LibraryController( importLocalArtwork = library.importLocalArtwork, importBarcodeIsbn = library.importBarcodeIsbn, scanForceModifiedTime = library.scanForceModifiedTime, - scanDeep = library.scanDeep + scanDeep = library.scanDeep, + convertToCbz = library.convertToCbz, ) ).toDto(includeRoot = principal.user.roleAdmin) } catch (e: Exception) { @@ -118,7 +119,8 @@ class LibraryController( importLocalArtwork = library.importLocalArtwork, importBarcodeIsbn = library.importBarcodeIsbn, scanForceModifiedTime = library.scanForceModifiedTime, - scanDeep = library.scanDeep + scanDeep = library.scanDeep, + convertToCbz = library.convertToCbz, ) libraryLifecycle.updateLibrary(toUpdate) } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) @@ -173,7 +175,8 @@ data class LibraryCreationDto( val importLocalArtwork: Boolean = true, val importBarcodeIsbn: Boolean = true, val scanForceModifiedTime: Boolean = false, - val scanDeep: Boolean = false + val scanDeep: Boolean = false, + val convertToCbz: Boolean = false, ) data class LibraryDto( @@ -189,7 +192,8 @@ data class LibraryDto( val importLocalArtwork: Boolean, val importBarcodeIsbn: Boolean, val scanForceModifiedTime: Boolean, - val scanDeep: Boolean + val scanDeep: Boolean, + val convertToCbz: Boolean, ) data class LibraryUpdateDto( @@ -204,7 +208,8 @@ data class LibraryUpdateDto( val importLocalArtwork: Boolean, val importBarcodeIsbn: Boolean, val scanForceModifiedTime: Boolean, - val scanDeep: Boolean + val scanDeep: Boolean, + val convertToCbz: Boolean, ) fun Library.toDto(includeRoot: Boolean) = LibraryDto( @@ -220,5 +225,6 @@ fun Library.toDto(includeRoot: Boolean) = LibraryDto( importLocalArtwork = importLocalArtwork, importBarcodeIsbn = importBarcodeIsbn, scanForceModifiedTime = scanForceModifiedTime, - scanDeep = scanDeep + scanDeep = scanDeep, + convertToCbz = convertToCbz, ) diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt index 53ef1e520..c5b7c71c2 100644 --- a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt @@ -63,6 +63,7 @@ class LibraryDaoTest( importComicInfoReadList = false, importBarcodeIsbn = false, importLocalArtwork = false, + convertToCbz = true, ) } @@ -85,6 +86,7 @@ class LibraryDaoTest( assertThat(modified.importComicInfoReadList).isEqualTo(updated.importComicInfoReadList) assertThat(modified.importBarcodeIsbn).isEqualTo(updated.importBarcodeIsbn) assertThat(modified.importLocalArtwork).isEqualTo(updated.importLocalArtwork) + assertThat(modified.convertToCbz).isEqualTo(updated.convertToCbz) } @Test