diff --git a/komga/src/flyway/resources/db/migration/sqlite/V20210505150806__library_repair_extensions.sql b/komga/src/flyway/resources/db/migration/sqlite/V20210505150806__library_repair_extensions.sql new file mode 100644 index 000000000..0e5663dcd --- /dev/null +++ b/komga/src/flyway/resources/db/migration/sqlite/V20210505150806__library_repair_extensions.sql @@ -0,0 +1,2 @@ +alter table library + add column REPAIR_EXTENSIONS boolean NOT NULL DEFAULT 0; diff --git a/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt b/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt index 90982e03a..fc02bc688 100644 --- a/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt +++ b/komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt @@ -50,4 +50,9 @@ sealed class Task(priority: Int = DEFAULT_PRIORITY) : Serializable { override fun uniqueId(): String = "CONVERT_BOOK_$bookId" override fun toString(): String = "ConvertBook(bookId='$bookId', priority='$priority')" } + + class RepairExtension(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { + override fun uniqueId(): String = "REPAIR_EXTENSION_$bookId" + override fun toString(): String = "RepairExtension(bookId='$bookId', priority='$priority')" + } } diff --git a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt index 31de55bd5..bfb912a7d 100644 --- a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt +++ b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt @@ -42,6 +42,7 @@ class TaskHandler( libraryRepository.findByIdOrNull(task.libraryId)?.let { library -> libraryContentLifecycle.scanRootFolder(library) taskReceiver.analyzeUnknownAndOutdatedBooks(library) + if (library.repairExtensions) taskReceiver.repairExtensions(library, LOWEST_PRIORITY) if (library.convertToCbz) taskReceiver.convertBooksToCbz(library, LOWEST_PRIORITY) } ?: logger.warn { "Cannot execute task $task: Library does not exist" } @@ -84,7 +85,12 @@ class TaskHandler( is Task.ConvertBook -> bookRepository.findByIdOrNull(task.bookId)?.let { book -> bookConverter.convertToCbz(book) - } + } ?: logger.warn { "Cannot execute task $task: Book does not exist" } + + is Task.RepairExtension -> + bookRepository.findByIdOrNull(task.bookId)?.let { book -> + bookConverter.repairExtension(book) + } ?: logger.warn { "Cannot execute task $task: Book does not exist" } } }.also { logger.info { "Task $task executed in $it" } diff --git a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt index 26d7e2c23..6ead9df20 100644 --- a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt +++ b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt @@ -9,7 +9,6 @@ import org.gotson.komga.domain.model.Library import org.gotson.komga.domain.model.Media import org.gotson.komga.domain.persistence.BookRepository import org.gotson.komga.domain.persistence.LibraryRepository -import org.gotson.komga.domain.persistence.MediaRepository import org.gotson.komga.domain.service.BookConverter import org.gotson.komga.infrastructure.jms.QUEUE_TASKS import org.gotson.komga.infrastructure.jms.QUEUE_TASKS_TYPE @@ -26,7 +25,6 @@ class TaskReceiver( private val jmsTemplate: JmsTemplate, private val libraryRepository: LibraryRepository, private val bookRepository: BookRepository, - private val mediaRepository: MediaRepository, private val bookConverter: BookConverter, ) { @@ -51,11 +49,17 @@ class TaskReceiver( } fun convertBooksToCbz(library: Library, priority: Int = DEFAULT_PRIORITY) { - mediaRepository.findBookIdsByMediaType(bookConverter.convertibleTypes).forEach { + bookConverter.getConvertibleBookIds(library).forEach { submitTask(Task.ConvertBook(it, priority)) } } + fun repairExtensions(library: Library, priority: Int = DEFAULT_PRIORITY) { + bookConverter.getMismatchedExtensionBookIds(library).forEach { + submitTask(Task.RepairExtension(it, priority)) + } + } + fun analyzeBook(bookId: String, priority: Int = DEFAULT_PRIORITY) { submitTask(Task.AnalyzeBook(bookId, priority)) } @@ -96,10 +100,6 @@ class TaskReceiver( submitTask(Task.ImportBook(sourceFile, seriesId, copyMode, destinationName, upgradeBookId, priority)) } - fun convertBook(bookId: String, priority: Int = DEFAULT_PRIORITY) { - submitTask(Task.ConvertBook(bookId, priority)) - } - private fun submitTask(task: Task) { logger.info { "Sending task: $task" } jmsTemplate.priority = task.priority diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt index 37be4ff80..cfb393bc5 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt @@ -19,6 +19,7 @@ data class Library( val importBarcodeIsbn: Boolean = true, val scanForceModifiedTime: Boolean = false, val scanDeep: Boolean = false, + val repairExtensions: Boolean = false, val convertToCbz: Boolean = false, val id: String = TsidCreator.getTsid256().toString(), diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt index d68f45f69..264eec997 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt @@ -19,6 +19,8 @@ interface BookRepository { fun findAllIdBySeriesIds(seriesIds: Collection): Collection fun findAllIdByLibraryId(libraryId: String): Collection fun findAllId(bookSearch: BookSearch, sort: Sort): Collection + fun findAllIdByLibraryIdAndMediaTypes(libraryId: String, mediaTypes: Collection): Collection + fun findAllIdByLibraryIdAndMismatchedExtension(libraryId: String, mediaType: String, extension: String): Collection fun insert(book: Book) fun insertMany(books: Collection) diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/MediaRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/MediaRepository.kt index acb5f8a4e..7f14f6da2 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/MediaRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/MediaRepository.kt @@ -4,7 +4,6 @@ import org.gotson.komga.domain.model.Media interface MediaRepository { fun findById(bookId: String): Media - fun findBookIdsByMediaType(mediaTypes: Collection): Collection fun insert(media: Media) fun insertMany(medias: Collection) diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/BookConverter.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/BookConverter.kt index 5ccb13b21..d28880763 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/BookConverter.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/BookConverter.kt @@ -4,10 +4,10 @@ import mu.KotlinLogging import org.apache.commons.compress.archivers.zip.ZipArchiveEntry import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream import org.apache.commons.io.FilenameUtils -import org.apache.tika.mime.MediaType import org.gotson.komga.domain.model.Book import org.gotson.komga.domain.model.BookConversionException import org.gotson.komga.domain.model.BookWithMedia +import org.gotson.komga.domain.model.Library import org.gotson.komga.domain.model.Media import org.gotson.komga.domain.model.MediaNotReadyException import org.gotson.komga.domain.model.MediaUnsupportedException @@ -20,6 +20,8 @@ import java.nio.file.FileAlreadyExistsException import java.util.zip.Deflater import kotlin.io.path.deleteIfExists import kotlin.io.path.exists +import kotlin.io.path.extension +import kotlin.io.path.moveTo import kotlin.io.path.nameWithoutExtension import kotlin.io.path.notExists import kotlin.io.path.outputStream @@ -37,15 +39,26 @@ class BookConverter( private val libraryRepository: LibraryRepository, ) { - val convertibleTypes = listOf("application/x-rar-compressed; version=4") + private val convertibleTypes = listOf("application/x-rar-compressed; version=4") - private val exclude = mutableListOf() + private val mediaTypeToExtension = mapOf( + "application/x-rar-compressed; version=4" to "cbr", + "application/zip" to "cbz", + "application/epub+zip" to "epub", + "application/pdf" to "pdf", + ) + + private val failedConversions = mutableListOf() + private val skippedRepairs = mutableListOf() + + fun getConvertibleBookIds(library: Library): Collection = + bookRepository.findAllIdByLibraryIdAndMediaTypes(library.id, convertibleTypes) fun convertToCbz(book: Book) { if (!libraryRepository.findById(book.libraryId).convertToCbz) return logger.info { "Book conversion is disabled for the library, it may have changed since the task was submitted, skipping" } - if (exclude.contains(book.id)) + if (failedConversions.contains(book.id)) return logger.info { "Book conversion already failed before, skipping" } if (book.path.notExists()) throw FileNotFoundException("File not found: ${book.path}") @@ -92,7 +105,7 @@ class BookConverter( convertedMedia.status != Media.Status.READY -> throw BookConversionException("Converted file could not be analyzed, aborting conversion") - convertedMedia.mediaType != MediaType.APPLICATION_ZIP.toString() + convertedMedia.mediaType != "application/zip" -> throw BookConversionException("Converted file is not a zip file, aborting conversion") !convertedMedia.pages.map { it.copy(fileName = FilenameUtils.getName(it.fileName)) } @@ -105,7 +118,7 @@ class BookConverter( } } catch (e: BookConversionException) { destinationPath.deleteIfExists() - exclude += book.id + failedConversions += book.id throw e } @@ -115,4 +128,50 @@ class BookConverter( bookRepository.update(convertedBook) mediaRepository.update(convertedMedia) } + + fun getMismatchedExtensionBookIds(library: Library): Collection = + mediaTypeToExtension.flatMap { (mediaType, extension) -> + bookRepository.findAllIdByLibraryIdAndMismatchedExtension(library.id, mediaType, extension) + } + + fun repairExtension(book: Book) { + if (!libraryRepository.findById(book.libraryId).repairExtensions) + return logger.info { "Repair extensions is disabled for the library, it may have changed since the task was submitted, skipping" } + + if (skippedRepairs.contains(book.id)) + return logger.info { "Extension repair has already been skipped before, skipping" } + + if (book.path.notExists()) throw FileNotFoundException("File not found: ${book.path}") + + val media = mediaRepository.findById(book.id) + + if (!mediaTypeToExtension.keys.contains(media.mediaType)) + throw MediaUnsupportedException("${media.mediaType} cannot be repaired. Must be one of ${mediaTypeToExtension.keys}") + + val actualExtension = book.path.extension + val correctExtension = mediaTypeToExtension[media.mediaType] + + if (correctExtension == actualExtension) { + logger.info { "MediaType (${media.mediaType}) and extension ($actualExtension) already match, skipping" } + skippedRepairs += book.id + } + + val destinationFilename = "${book.path.nameWithoutExtension}.$correctExtension" + val destinationPath = book.path.parent.resolve(destinationFilename) + if (destinationPath.exists()) + throw FileAlreadyExistsException("Destination file already exists: $destinationPath") + + logger.info { "Renaming ${book.path} to $destinationPath" } + book.path.moveTo(destinationPath) + + val repairedBook = fileSystemScanner.scanFile(destinationPath) + ?.copy( + id = book.id, + seriesId = book.seriesId, + libraryId = book.libraryId + ) + ?: throw IllegalStateException("Repaired book could not be scanned: $destinationFilename") + + bookRepository.update(repairedBook) + } } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt index 7f0a61428..b6cb75e11 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt @@ -141,6 +141,23 @@ class BookDao( .fetch(b.ID) } + override fun findAllIdByLibraryIdAndMediaTypes(libraryId: String, mediaTypes: Collection): Collection = + dsl.select(b.ID) + .from(b) + .leftJoin(m).on(b.ID.eq(m.BOOK_ID)) + .where(b.LIBRARY_ID.eq(libraryId)) + .and(m.MEDIA_TYPE.`in`(mediaTypes)) + .fetch(b.ID) + + override fun findAllIdByLibraryIdAndMismatchedExtension(libraryId: String, mediaType: String, extension: String): Collection = + dsl.select(b.ID) + .from(b) + .leftJoin(m).on(b.ID.eq(m.BOOK_ID)) + .where(b.LIBRARY_ID.eq(libraryId)) + .and(m.MEDIA_TYPE.eq(mediaType)) + .and(b.URL.notLike("%.$extension")) + .fetch(b.ID) + override fun insert(book: Book) { insertMany(listOf(book)) } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt index a8588560f..ad639ca55 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt @@ -75,6 +75,7 @@ class LibraryDao( .set(l.IMPORT_BARCODE_ISBN, library.importBarcodeIsbn) .set(l.SCAN_FORCE_MODIFIED_TIME, library.scanForceModifiedTime) .set(l.SCAN_DEEP, library.scanDeep) + .set(l.REPAIR_EXTENSIONS, library.repairExtensions) .set(l.CONVERT_TO_CBZ, library.convertToCbz) .execute() } @@ -93,6 +94,7 @@ class LibraryDao( .set(l.IMPORT_BARCODE_ISBN, library.importBarcodeIsbn) .set(l.SCAN_FORCE_MODIFIED_TIME, library.scanForceModifiedTime) .set(l.SCAN_DEEP, library.scanDeep) + .set(l.REPAIR_EXTENSIONS, library.repairExtensions) .set(l.CONVERT_TO_CBZ, library.convertToCbz) .set(l.LAST_MODIFIED_DATE, LocalDateTime.now(ZoneId.of("Z"))) .where(l.ID.eq(library.id)) @@ -115,6 +117,7 @@ class LibraryDao( importBarcodeIsbn = importBarcodeIsbn, scanForceModifiedTime = scanForceModifiedTime, scanDeep = scanDeep, + repairExtensions = repairExtensions, convertToCbz = convertToCbz, id = id, createdDate = createdDate.toCurrentTimeZone(), diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt index 429085947..7a8ea7468 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt @@ -26,12 +26,6 @@ class MediaDao( override fun findById(bookId: String): Media = find(dsl, bookId) - override fun findBookIdsByMediaType(mediaTypes: Collection): Collection = - dsl.select(m.BOOK_ID) - .from(m) - .where(m.MEDIA_TYPE.`in`(mediaTypes)) - .fetch(m.BOOK_ID) - private fun find(dsl: DSLContext, bookId: String): Media = dsl.select(*groupFields) .from(m) diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/LibraryController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/LibraryController.kt index 9999484c8..ad26d7124 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/LibraryController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/rest/LibraryController.kt @@ -84,6 +84,7 @@ class LibraryController( importBarcodeIsbn = library.importBarcodeIsbn, scanForceModifiedTime = library.scanForceModifiedTime, scanDeep = library.scanDeep, + repairExtensions = library.repairExtensions, convertToCbz = library.convertToCbz, ) ).toDto(includeRoot = principal.user.roleAdmin) @@ -120,6 +121,7 @@ class LibraryController( importBarcodeIsbn = library.importBarcodeIsbn, scanForceModifiedTime = library.scanForceModifiedTime, scanDeep = library.scanDeep, + repairExtensions = library.repairExtensions, convertToCbz = library.convertToCbz, ) libraryLifecycle.updateLibrary(toUpdate) @@ -176,6 +178,7 @@ data class LibraryCreationDto( val importBarcodeIsbn: Boolean = true, val scanForceModifiedTime: Boolean = false, val scanDeep: Boolean = false, + val repairExtensions: Boolean = false, val convertToCbz: Boolean = false, ) @@ -193,6 +196,7 @@ data class LibraryDto( val importBarcodeIsbn: Boolean, val scanForceModifiedTime: Boolean, val scanDeep: Boolean, + val repairExtensions: Boolean, val convertToCbz: Boolean, ) @@ -209,6 +213,7 @@ data class LibraryUpdateDto( val importBarcodeIsbn: Boolean, val scanForceModifiedTime: Boolean, val scanDeep: Boolean, + val repairExtensions: Boolean, val convertToCbz: Boolean, ) @@ -226,5 +231,6 @@ fun Library.toDto(includeRoot: Boolean) = LibraryDto( importBarcodeIsbn = importBarcodeIsbn, scanForceModifiedTime = scanForceModifiedTime, scanDeep = scanDeep, + repairExtensions = repairExtensions, convertToCbz = convertToCbz, ) diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt index c5b7c71c2..8e36e58a8 100644 --- a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt @@ -63,6 +63,7 @@ class LibraryDaoTest( importComicInfoReadList = false, importBarcodeIsbn = false, importLocalArtwork = false, + repairExtensions = true, convertToCbz = true, ) } @@ -86,6 +87,7 @@ class LibraryDaoTest( assertThat(modified.importComicInfoReadList).isEqualTo(updated.importComicInfoReadList) assertThat(modified.importBarcodeIsbn).isEqualTo(updated.importBarcodeIsbn) assertThat(modified.importLocalArtwork).isEqualTo(updated.importLocalArtwork) + assertThat(modified.repairExtensions).isEqualTo(updated.repairExtensions) assertThat(modified.convertToCbz).isEqualTo(updated.convertToCbz) }