feat: repair file extensions

komga/src/flyway/resources/db/migration/sqlite/V20210505150806__library_repair_extensions.sql

@@ -0,0 +1,2 @@
+alter table library
+    add column REPAIR_EXTENSIONS boolean NOT NULL DEFAULT 0;

komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt

@@ -50,4 +50,9 @@ sealed class Task(priority: Int = DEFAULT_PRIORITY) : Serializable {
     override fun uniqueId(): String = "CONVERT_BOOK_$bookId"
     override fun toString(): String = "ConvertBook(bookId='$bookId', priority='$priority')"
   }
+
+  class RepairExtension(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) {
+    override fun uniqueId(): String = "REPAIR_EXTENSION_$bookId"
+    override fun toString(): String = "RepairExtension(bookId='$bookId', priority='$priority')"
+  }
 }

komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt

@@ -42,6 +42,7 @@ class TaskHandler(
             libraryRepository.findByIdOrNull(task.libraryId)?.let { library ->
               libraryContentLifecycle.scanRootFolder(library)
               taskReceiver.analyzeUnknownAndOutdatedBooks(library)
+              if (library.repairExtensions) taskReceiver.repairExtensions(library, LOWEST_PRIORITY)
               if (library.convertToCbz) taskReceiver.convertBooksToCbz(library, LOWEST_PRIORITY)
             } ?: logger.warn { "Cannot execute task $task: Library does not exist" }
 
@@ -84,7 +85,12 @@ class TaskHandler(
           is Task.ConvertBook ->
             bookRepository.findByIdOrNull(task.bookId)?.let { book ->
               bookConverter.convertToCbz(book)
-            }
+            } ?: logger.warn { "Cannot execute task $task: Book does not exist" }
+
+          is Task.RepairExtension ->
+            bookRepository.findByIdOrNull(task.bookId)?.let { book ->
+              bookConverter.repairExtension(book)
+            } ?: logger.warn { "Cannot execute task $task: Book does not exist" }
         }
       }.also {
         logger.info { "Task $task executed in $it" }

komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt

@@ -9,7 +9,6 @@ import org.gotson.komga.domain.model.Library
 import org.gotson.komga.domain.model.Media
 import org.gotson.komga.domain.persistence.BookRepository
 import org.gotson.komga.domain.persistence.LibraryRepository
-import org.gotson.komga.domain.persistence.MediaRepository
 import org.gotson.komga.domain.service.BookConverter
 import org.gotson.komga.infrastructure.jms.QUEUE_TASKS
 import org.gotson.komga.infrastructure.jms.QUEUE_TASKS_TYPE
@@ -26,7 +25,6 @@ class TaskReceiver(
   private val jmsTemplate: JmsTemplate,
   private val libraryRepository: LibraryRepository,
   private val bookRepository: BookRepository,
-  private val mediaRepository: MediaRepository,
   private val bookConverter: BookConverter,
 ) {
 
@@ -51,11 +49,17 @@ class TaskReceiver(
   }
 
   fun convertBooksToCbz(library: Library, priority: Int = DEFAULT_PRIORITY) {
-    mediaRepository.findBookIdsByMediaType(bookConverter.convertibleTypes).forEach {
+    bookConverter.getConvertibleBookIds(library).forEach {
       submitTask(Task.ConvertBook(it, priority))
     }
   }
 
+  fun repairExtensions(library: Library, priority: Int = DEFAULT_PRIORITY) {
+    bookConverter.getMismatchedExtensionBookIds(library).forEach {
+      submitTask(Task.RepairExtension(it, priority))
+    }
+  }
+
   fun analyzeBook(bookId: String, priority: Int = DEFAULT_PRIORITY) {
     submitTask(Task.AnalyzeBook(bookId, priority))
   }
@@ -96,10 +100,6 @@ class TaskReceiver(
     submitTask(Task.ImportBook(sourceFile, seriesId, copyMode, destinationName, upgradeBookId, priority))
   }
 
-  fun convertBook(bookId: String, priority: Int = DEFAULT_PRIORITY) {
-    submitTask(Task.ConvertBook(bookId, priority))
-  }
-
   private fun submitTask(task: Task) {
     logger.info { "Sending task: $task" }
     jmsTemplate.priority = task.priority

komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt

@@ -19,6 +19,7 @@ data class Library(
   val importBarcodeIsbn: Boolean = true,
   val scanForceModifiedTime: Boolean = false,
   val scanDeep: Boolean = false,
+  val repairExtensions: Boolean = false,
   val convertToCbz: Boolean = false,
 
   val id: String = TsidCreator.getTsid256().toString(),

komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt

@@ -19,6 +19,8 @@ interface BookRepository {
   fun findAllIdBySeriesIds(seriesIds: Collection<String>): Collection<String>
   fun findAllIdByLibraryId(libraryId: String): Collection<String>
   fun findAllId(bookSearch: BookSearch, sort: Sort): Collection<String>
+  fun findAllIdByLibraryIdAndMediaTypes(libraryId: String, mediaTypes: Collection<String>): Collection<String>
+  fun findAllIdByLibraryIdAndMismatchedExtension(libraryId: String, mediaType: String, extension: String): Collection<String>
 
   fun insert(book: Book)
   fun insertMany(books: Collection<Book>)

komga/src/main/kotlin/org/gotson/komga/domain/persistence/MediaRepository.kt

@@ -4,7 +4,6 @@ import org.gotson.komga.domain.model.Media
 
 interface MediaRepository {
   fun findById(bookId: String): Media
-  fun findBookIdsByMediaType(mediaTypes: Collection<String>): Collection<String>
 
   fun insert(media: Media)
   fun insertMany(medias: Collection<Media>)

komga/src/main/kotlin/org/gotson/komga/domain/service/BookConverter.kt

@@ -4,10 +4,10 @@ import mu.KotlinLogging
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
 import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
 import org.apache.commons.io.FilenameUtils
-import org.apache.tika.mime.MediaType
 import org.gotson.komga.domain.model.Book
 import org.gotson.komga.domain.model.BookConversionException
 import org.gotson.komga.domain.model.BookWithMedia
+import org.gotson.komga.domain.model.Library
 import org.gotson.komga.domain.model.Media
 import org.gotson.komga.domain.model.MediaNotReadyException
 import org.gotson.komga.domain.model.MediaUnsupportedException
@@ -20,6 +20,8 @@ import java.nio.file.FileAlreadyExistsException
 import java.util.zip.Deflater
 import kotlin.io.path.deleteIfExists
 import kotlin.io.path.exists
+import kotlin.io.path.extension
+import kotlin.io.path.moveTo
 import kotlin.io.path.nameWithoutExtension
 import kotlin.io.path.notExists
 import kotlin.io.path.outputStream
@@ -37,15 +39,26 @@ class BookConverter(
   private val libraryRepository: LibraryRepository,
 ) {
 
-  val convertibleTypes = listOf("application/x-rar-compressed; version=4")
+  private val convertibleTypes = listOf("application/x-rar-compressed; version=4")
 
-  private val exclude = mutableListOf<String>()
+  private val mediaTypeToExtension = mapOf(
+    "application/x-rar-compressed; version=4" to "cbr",
+    "application/zip" to "cbz",
+    "application/epub+zip" to "epub",
+    "application/pdf" to "pdf",
+  )
+
+  private val failedConversions = mutableListOf<String>()
+  private val skippedRepairs = mutableListOf<String>()
+
+  fun getConvertibleBookIds(library: Library): Collection<String> =
+    bookRepository.findAllIdByLibraryIdAndMediaTypes(library.id, convertibleTypes)
 
   fun convertToCbz(book: Book) {
     if (!libraryRepository.findById(book.libraryId).convertToCbz)
       return logger.info { "Book conversion is disabled for the library, it may have changed since the task was submitted, skipping" }
 
-    if (exclude.contains(book.id))
+    if (failedConversions.contains(book.id))
       return logger.info { "Book conversion already failed before, skipping" }
 
     if (book.path.notExists()) throw FileNotFoundException("File not found: ${book.path}")
@@ -92,7 +105,7 @@ class BookConverter(
         convertedMedia.status != Media.Status.READY
         -> throw BookConversionException("Converted file could not be analyzed, aborting conversion")
 
-        convertedMedia.mediaType != MediaType.APPLICATION_ZIP.toString()
+        convertedMedia.mediaType != "application/zip"
         -> throw BookConversionException("Converted file is not a zip file, aborting conversion")
 
         !convertedMedia.pages.map { it.copy(fileName = FilenameUtils.getName(it.fileName)) }
@@ -105,7 +118,7 @@ class BookConverter(
       }
     } catch (e: BookConversionException) {
       destinationPath.deleteIfExists()
-      exclude += book.id
+      failedConversions += book.id
       throw e
     }
 
@@ -115,4 +128,50 @@ class BookConverter(
     bookRepository.update(convertedBook)
     mediaRepository.update(convertedMedia)
   }
+
+  fun getMismatchedExtensionBookIds(library: Library): Collection<String> =
+    mediaTypeToExtension.flatMap { (mediaType, extension) ->
+      bookRepository.findAllIdByLibraryIdAndMismatchedExtension(library.id, mediaType, extension)
+    }
+
+  fun repairExtension(book: Book) {
+    if (!libraryRepository.findById(book.libraryId).repairExtensions)
+      return logger.info { "Repair extensions is disabled for the library, it may have changed since the task was submitted, skipping" }
+
+    if (skippedRepairs.contains(book.id))
+      return logger.info { "Extension repair has already been skipped before, skipping" }
+
+    if (book.path.notExists()) throw FileNotFoundException("File not found: ${book.path}")
+
+    val media = mediaRepository.findById(book.id)
+
+    if (!mediaTypeToExtension.keys.contains(media.mediaType))
+      throw MediaUnsupportedException("${media.mediaType} cannot be repaired. Must be one of ${mediaTypeToExtension.keys}")
+
+    val actualExtension = book.path.extension
+    val correctExtension = mediaTypeToExtension[media.mediaType]
+
+    if (correctExtension == actualExtension) {
+      logger.info { "MediaType (${media.mediaType}) and extension ($actualExtension) already match, skipping" }
+      skippedRepairs += book.id
+    }
+
+    val destinationFilename = "${book.path.nameWithoutExtension}.$correctExtension"
+    val destinationPath = book.path.parent.resolve(destinationFilename)
+    if (destinationPath.exists())
+      throw FileAlreadyExistsException("Destination file already exists: $destinationPath")
+
+    logger.info { "Renaming ${book.path} to $destinationPath" }
+    book.path.moveTo(destinationPath)
+
+    val repairedBook = fileSystemScanner.scanFile(destinationPath)
+      ?.copy(
+        id = book.id,
+        seriesId = book.seriesId,
+        libraryId = book.libraryId
+      )
+      ?: throw IllegalStateException("Repaired book could not be scanned: $destinationFilename")
+
+    bookRepository.update(repairedBook)
+  }
 }

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt

@@ -141,6 +141,23 @@ class BookDao(
       .fetch(b.ID)
   }
 
+  override fun findAllIdByLibraryIdAndMediaTypes(libraryId: String, mediaTypes: Collection<String>): Collection<String> =
+    dsl.select(b.ID)
+      .from(b)
+      .leftJoin(m).on(b.ID.eq(m.BOOK_ID))
+      .where(b.LIBRARY_ID.eq(libraryId))
+      .and(m.MEDIA_TYPE.`in`(mediaTypes))
+      .fetch(b.ID)
+
+  override fun findAllIdByLibraryIdAndMismatchedExtension(libraryId: String, mediaType: String, extension: String): Collection<String> =
+    dsl.select(b.ID)
+      .from(b)
+      .leftJoin(m).on(b.ID.eq(m.BOOK_ID))
+      .where(b.LIBRARY_ID.eq(libraryId))
+      .and(m.MEDIA_TYPE.eq(mediaType))
+      .and(b.URL.notLike("%.$extension"))
+      .fetch(b.ID)
+
   override fun insert(book: Book) {
     insertMany(listOf(book))
   }

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt

@@ -75,6 +75,7 @@ class LibraryDao(
       .set(l.IMPORT_BARCODE_ISBN, library.importBarcodeIsbn)
       .set(l.SCAN_FORCE_MODIFIED_TIME, library.scanForceModifiedTime)
       .set(l.SCAN_DEEP, library.scanDeep)
+      .set(l.REPAIR_EXTENSIONS, library.repairExtensions)
       .set(l.CONVERT_TO_CBZ, library.convertToCbz)
       .execute()
   }
@@ -93,6 +94,7 @@ class LibraryDao(
       .set(l.IMPORT_BARCODE_ISBN, library.importBarcodeIsbn)
       .set(l.SCAN_FORCE_MODIFIED_TIME, library.scanForceModifiedTime)
       .set(l.SCAN_DEEP, library.scanDeep)
+      .set(l.REPAIR_EXTENSIONS, library.repairExtensions)
       .set(l.CONVERT_TO_CBZ, library.convertToCbz)
       .set(l.LAST_MODIFIED_DATE, LocalDateTime.now(ZoneId.of("Z")))
       .where(l.ID.eq(library.id))
@@ -115,6 +117,7 @@ class LibraryDao(
       importBarcodeIsbn = importBarcodeIsbn,
       scanForceModifiedTime = scanForceModifiedTime,
       scanDeep = scanDeep,
+      repairExtensions = repairExtensions,
       convertToCbz = convertToCbz,
       id = id,
       createdDate = createdDate.toCurrentTimeZone(),

komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/MediaDao.kt

@@ -26,12 +26,6 @@ class MediaDao(
   override fun findById(bookId: String): Media =
     find(dsl, bookId)
 
-  override fun findBookIdsByMediaType(mediaTypes: Collection<String>): Collection<String> =
-    dsl.select(m.BOOK_ID)
-      .from(m)
-      .where(m.MEDIA_TYPE.`in`(mediaTypes))
-      .fetch(m.BOOK_ID)
-
   private fun find(dsl: DSLContext, bookId: String): Media =
     dsl.select(*groupFields)
       .from(m)

komga/src/main/kotlin/org/gotson/komga/interfaces/rest/LibraryController.kt

@@ -84,6 +84,7 @@ class LibraryController(
           importBarcodeIsbn = library.importBarcodeIsbn,
           scanForceModifiedTime = library.scanForceModifiedTime,
           scanDeep = library.scanDeep,
+          repairExtensions = library.repairExtensions,
           convertToCbz = library.convertToCbz,
         )
       ).toDto(includeRoot = principal.user.roleAdmin)
@@ -120,6 +121,7 @@ class LibraryController(
         importBarcodeIsbn = library.importBarcodeIsbn,
         scanForceModifiedTime = library.scanForceModifiedTime,
         scanDeep = library.scanDeep,
+        repairExtensions = library.repairExtensions,
         convertToCbz = library.convertToCbz,
       )
       libraryLifecycle.updateLibrary(toUpdate)
@@ -176,6 +178,7 @@ data class LibraryCreationDto(
   val importBarcodeIsbn: Boolean = true,
   val scanForceModifiedTime: Boolean = false,
   val scanDeep: Boolean = false,
+  val repairExtensions: Boolean = false,
   val convertToCbz: Boolean = false,
 )
 
@@ -193,6 +196,7 @@ data class LibraryDto(
   val importBarcodeIsbn: Boolean,
   val scanForceModifiedTime: Boolean,
   val scanDeep: Boolean,
+  val repairExtensions: Boolean,
   val convertToCbz: Boolean,
 )
 
@@ -209,6 +213,7 @@ data class LibraryUpdateDto(
   val importBarcodeIsbn: Boolean,
   val scanForceModifiedTime: Boolean,
   val scanDeep: Boolean,
+  val repairExtensions: Boolean,
   val convertToCbz: Boolean,
 )
 
@@ -226,5 +231,6 @@ fun Library.toDto(includeRoot: Boolean) = LibraryDto(
   importBarcodeIsbn = importBarcodeIsbn,
   scanForceModifiedTime = scanForceModifiedTime,
   scanDeep = scanDeep,
+  repairExtensions = repairExtensions,
   convertToCbz = convertToCbz,
 )

komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt

@@ -63,6 +63,7 @@ class LibraryDaoTest(
         importComicInfoReadList = false,
         importBarcodeIsbn = false,
         importLocalArtwork = false,
+        repairExtensions = true,
         convertToCbz = true,
       )
     }
@@ -86,6 +87,7 @@ class LibraryDaoTest(
     assertThat(modified.importComicInfoReadList).isEqualTo(updated.importComicInfoReadList)
     assertThat(modified.importBarcodeIsbn).isEqualTo(updated.importBarcodeIsbn)
     assertThat(modified.importLocalArtwork).isEqualTo(updated.importLocalArtwork)
+    assertThat(modified.repairExtensions).isEqualTo(updated.repairExtensions)
     assertThat(modified.convertToCbz).isEqualTo(updated.convertToCbz)
   }