Backup_Repos/komga
1
0
Fork 0
mirror of https://github.com/gotson/komga.git synced 2026-05-07 12:01:40 +02:00
Code Issues Projects Releases Wiki Activity

feat: tasks concurrency (configurable)

Browse source
This commit is contained in:
Gauthier Roebroeck 2021-09-19 22:48:53 +08:00
parent 39f686bebe
commit 2fd95e5a7f
17 changed files with 104 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
komga/src/main/kotlin/org/gotson/komga/application/tasks/Task.kt
View file

@ -10,7 +10,7 @@ const val HIGH_PRIORITY = 6
const val DEFAULT_PRIORITY = 4 const val DEFAULT_PRIORITY = 4
const val LOWEST_PRIORITY = 0 const val LOWEST_PRIORITY = 0
sealed class Task(priority: Int = DEFAULT_PRIORITY) : Serializable { sealed class Task(priority: Int = DEFAULT_PRIORITY, val groupId: String? = null) : Serializable {
abstract fun uniqueId(): String abstract fun uniqueId(): String
val priority = priority.coerceIn(0, 9) val priority = priority.coerceIn(0, 9)
@ -24,63 +24,63 @@ sealed class Task(priority: Int = DEFAULT_PRIORITY) : Serializable {
override fun toString(): String = "EmptyTrash(libraryId='$libraryId', priority='$priority')" override fun toString(): String = "EmptyTrash(libraryId='$libraryId', priority='$priority')"
} }
class AnalyzeBook(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class AnalyzeBook(val bookId: String, priority: Int = DEFAULT_PRIORITY, groupId: String) : Task(priority, groupId.takeLast(1)) {
override fun uniqueId() = "ANALYZE_BOOK_$bookId" override fun uniqueId() = "ANALYZE_BOOK_$bookId"
override fun toString(): String = "AnalyzeBook(bookId='$bookId', priority='$priority')" override fun toString(): String = "AnalyzeBook(bookId='$bookId', priority='$priority')"
} }
class GenerateBookThumbnail(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class GenerateBookThumbnail(val bookId: String, priority: Int = DEFAULT_PRIORITY, groupId: String) : Task(priority, groupId.takeLast(1)) {
override fun uniqueId() = "GENERATE_BOOK_THUMBNAIL_$bookId" override fun uniqueId() = "GENERATE_BOOK_THUMBNAIL_$bookId"
override fun toString(): String = "GenerateBookThumbnail(bookId='$bookId', priority='$priority')" override fun toString(): String = "GenerateBookThumbnail(bookId='$bookId', priority='$priority')"
} }
class RefreshBookMetadata(val bookId: String, val capabilities: Set<BookMetadataPatchCapability>, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class RefreshBookMetadata(val bookId: String, val capabilities: Set<BookMetadataPatchCapability>, priority: Int = DEFAULT_PRIORITY, groupId: String) : Task(priority, groupId.takeLast(1)) {
override fun uniqueId() = "REFRESH_BOOK_METADATA_$bookId" override fun uniqueId() = "REFRESH_BOOK_METADATA_$bookId"
override fun toString(): String = "RefreshBookMetadata(bookId='$bookId', capabilities=$capabilities, priority='$priority')" override fun toString(): String = "RefreshBookMetadata(bookId='$bookId', capabilities=$capabilities, priority='$priority')"
} }
class HashBook(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class HashBook(val bookId: String, priority: Int = DEFAULT_PRIORITY, groupId: String) : Task(priority, groupId.takeLast(1)) {
override fun uniqueId() = "HASH_BOOK_$bookId" override fun uniqueId() = "HASH_BOOK_$bookId"
override fun toString(): String = "HashBook(bookId='$bookId', priority='$priority')" override fun toString(): String = "HashBook(bookId='$bookId', priority='$priority')"
} }
class HashBookPages(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class HashBookPages(val bookId: String, priority: Int = DEFAULT_PRIORITY, groupId: String) : Task(priority, groupId.takeLast(1)) {
override fun uniqueId() = "HASH_BOOK_PAGES_$bookId" override fun uniqueId() = "HASH_BOOK_PAGES_$bookId"
override fun toString(): String = "HashBookPages(bookId='$bookId', priority='$priority')" override fun toString(): String = "HashBookPages(bookId='$bookId', priority='$priority')"
} }
class RefreshSeriesMetadata(val seriesId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class RefreshSeriesMetadata(val seriesId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority, seriesId.takeLast(1)) {
override fun uniqueId() = "REFRESH_SERIES_METADATA_$seriesId" override fun uniqueId() = "REFRESH_SERIES_METADATA_$seriesId"
override fun toString(): String = "RefreshSeriesMetadata(seriesId='$seriesId', priority='$priority')" override fun toString(): String = "RefreshSeriesMetadata(seriesId='$seriesId', priority='$priority')"
} }
class AggregateSeriesMetadata(val seriesId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class AggregateSeriesMetadata(val seriesId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority, seriesId.takeLast(1)) {
override fun uniqueId() = "AGGREGATE_SERIES_METADATA_$seriesId" override fun uniqueId() = "AGGREGATE_SERIES_METADATA_$seriesId"
override fun toString(): String = "AggregateSeriesMetadata(seriesId='$seriesId', priority='$priority')" override fun toString(): String = "AggregateSeriesMetadata(seriesId='$seriesId', priority='$priority')"
} }
class RefreshBookLocalArtwork(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class RefreshBookLocalArtwork(val bookId: String, priority: Int = DEFAULT_PRIORITY, groupId: String) : Task(priority, groupId.takeLast(1)) {
override fun uniqueId(): String = "REFRESH_BOOK_LOCAL_ARTWORK_$bookId" override fun uniqueId(): String = "REFRESH_BOOK_LOCAL_ARTWORK_$bookId"
override fun toString(): String = "RefreshBookLocalArtwork(bookId='$bookId', priority='$priority')" override fun toString(): String = "RefreshBookLocalArtwork(bookId='$bookId', priority='$priority')"
} }
class RefreshSeriesLocalArtwork(val seriesId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class RefreshSeriesLocalArtwork(val seriesId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority, seriesId.takeLast(1)) {
override fun uniqueId(): String = "REFRESH_SERIES_LOCAL_ARTWORK_$seriesId" override fun uniqueId(): String = "REFRESH_SERIES_LOCAL_ARTWORK_$seriesId"
override fun toString(): String = "RefreshSeriesLocalArtwork(seriesId=$seriesId, priority='$priority')" override fun toString(): String = "RefreshSeriesLocalArtwork(seriesId=$seriesId, priority='$priority')"
} }
class ImportBook(val sourceFile: String, val seriesId: String, val copyMode: CopyMode, val destinationName: String?, val upgradeBookId: String?, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class ImportBook(val sourceFile: String, val seriesId: String, val copyMode: CopyMode, val destinationName: String?, val upgradeBookId: String?, priority: Int = DEFAULT_PRIORITY) : Task(priority, seriesId.takeLast(1)) {
override fun uniqueId(): String = "IMPORT_BOOK_${seriesId}_$sourceFile" override fun uniqueId(): String = "IMPORT_BOOK_${seriesId}_$sourceFile"
override fun toString(): String = override fun toString(): String =
"ImportBook(sourceFile='$sourceFile', seriesId='$seriesId', copyMode=$copyMode, destinationName=$destinationName, upgradeBookId=$upgradeBookId, priority='$priority')" "ImportBook(sourceFile='$sourceFile', seriesId='$seriesId', copyMode=$copyMode, destinationName=$destinationName, upgradeBookId=$upgradeBookId, priority='$priority')"
} }
class ConvertBook(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class ConvertBook(val bookId: String, priority: Int = DEFAULT_PRIORITY, groupId: String) : Task(priority, groupId.takeLast(1)) {
override fun uniqueId(): String = "CONVERT_BOOK_$bookId" override fun uniqueId(): String = "CONVERT_BOOK_$bookId"
override fun toString(): String = "ConvertBook(bookId='$bookId', priority='$priority')" override fun toString(): String = "ConvertBook(bookId='$bookId', priority='$priority')"
} }
class RepairExtension(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class RepairExtension(val bookId: String, priority: Int = DEFAULT_PRIORITY, groupId: String) : Task(priority, groupId.takeLast(1)) {
override fun uniqueId(): String = "REPAIR_EXTENSION_$bookId" override fun uniqueId(): String = "REPAIR_EXTENSION_$bookId"
override fun toString(): String = "RepairExtension(bookId='$bookId', priority='$priority')" override fun toString(): String = "RepairExtension(bookId='$bookId', priority='$priority')"
} }

8
komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt
View file

@ -40,7 +40,7 @@ class TaskHandler(
private val searchIndexLifecycle: SearchIndexLifecycle, private val searchIndexLifecycle: SearchIndexLifecycle,
) { ) {
@JmsListener(destination = QUEUE_TASKS, selector = QUEUE_TASKS_SELECTOR, containerFactory = QUEUE_FACTORY) @JmsListener(destination = QUEUE_TASKS, selector = QUEUE_TASKS_SELECTOR, containerFactory = QUEUE_FACTORY, concurrency = "#{@komgaProperties.taskConsumers}-#{@komgaProperties.taskConsumersMax}")
fun handleTask(task: Task) { fun handleTask(task: Task) {
logger.info { "Executing task: $task" } logger.info { "Executing task: $task" }
try { try {
@ -65,8 +65,8 @@ class TaskHandler(
is Task.AnalyzeBook -> is Task.AnalyzeBook ->
bookRepository.findByIdOrNull(task.bookId)?.let { book -> bookRepository.findByIdOrNull(task.bookId)?.let { book ->
if (bookLifecycle.analyzeAndPersist(book)) { if (bookLifecycle.analyzeAndPersist(book)) {
taskReceiver.generateBookThumbnail(book.id, priority = task.priority + 1) taskReceiver.generateBookThumbnail(book, priority = task.priority + 1)
taskReceiver.refreshBookMetadata(book.id, priority = task.priority + 1) taskReceiver.refreshBookMetadata(book, priority = task.priority + 1)
} }
} ?: logger.warn { "Cannot execute task $task: Book does not exist" } } ?: logger.warn { "Cannot execute task $task: Book does not exist" }
@ -105,7 +105,7 @@ class TaskHandler(
is Task.ImportBook -> is Task.ImportBook ->
seriesRepository.findByIdOrNull(task.seriesId)?.let { series -> seriesRepository.findByIdOrNull(task.seriesId)?.let { series ->
val importedBook = bookImporter.importBook(Paths.get(task.sourceFile), series, task.copyMode, task.destinationName, task.upgradeBookId) val importedBook = bookImporter.importBook(Paths.get(task.sourceFile), series, task.copyMode, task.destinationName, task.upgradeBookId)
taskReceiver.analyzeBook(importedBook.id, priority = task.priority + 1) taskReceiver.analyzeBook(importedBook, priority = task.priority + 1)
} ?: logger.warn { "Cannot execute task $task: Series does not exist" } } ?: logger.warn { "Cannot execute task $task: Series does not exist" }
is Task.ConvertBook -> is Task.ConvertBook ->

39
komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskReceiver.kt
View file

@ -1,6 +1,7 @@
package org.gotson.komga.application.tasks package org.gotson.komga.application.tasks
import mu.KotlinLogging import mu.KotlinLogging
import org.gotson.komga.domain.model.Book
import org.gotson.komga.domain.model.BookMetadataPatchCapability import org.gotson.komga.domain.model.BookMetadataPatchCapability
import org.gotson.komga.domain.model.BookSearch import org.gotson.komga.domain.model.BookSearch
import org.gotson.komga.domain.model.CopyMode import org.gotson.komga.domain.model.CopyMode
@ -16,6 +17,7 @@ import org.gotson.komga.infrastructure.jms.QUEUE_TASKS
import org.gotson.komga.infrastructure.jms.QUEUE_TASKS_TYPE import org.gotson.komga.infrastructure.jms.QUEUE_TASKS_TYPE
import org.gotson.komga.infrastructure.jms.QUEUE_TYPE import org.gotson.komga.infrastructure.jms.QUEUE_TYPE
import org.gotson.komga.infrastructure.jms.QUEUE_UNIQUE_ID import org.gotson.komga.infrastructure.jms.QUEUE_UNIQUE_ID
import org.gotson.komga.infrastructure.jooq.UnpagedSorted
import org.gotson.komga.infrastructure.search.LuceneEntity import org.gotson.komga.infrastructure.search.LuceneEntity
import org.springframework.data.domain.Sort import org.springframework.data.domain.Sort
import org.springframework.jms.core.JmsTemplate import org.springframework.jms.core.JmsTemplate
@ -54,59 +56,59 @@ class TaskReceiver(
} }
fun analyzeUnknownAndOutdatedBooks(library: Library) { fun analyzeUnknownAndOutdatedBooks(library: Library) {
bookRepository.findAllIds( bookRepository.findAll(
BookSearch( BookSearch(
libraryIds = listOf(library.id), libraryIds = listOf(library.id),
mediaStatus = listOf(Media.Status.UNKNOWN, Media.Status.OUTDATED), mediaStatus = listOf(Media.Status.UNKNOWN, Media.Status.OUTDATED),
), ),
Sort.by(Sort.Order.asc("seriesId"), Sort.Order.asc("number")), UnpagedSorted(Sort.by(Sort.Order.asc("seriesId"), Sort.Order.asc("number")))
).forEach { ).forEach {
submitTask(Task.AnalyzeBook(it)) submitTask(Task.AnalyzeBook(it.id, groupId = it.seriesId))
} }
} }
fun hashBooksWithoutHash(library: Library) { fun hashBooksWithoutHash(library: Library) {
if (library.hashFiles) if (library.hashFiles)
bookRepository.findAllIdsByLibraryIdAndWithEmptyHash(library.id).forEach { bookRepository.findAllByLibraryIdAndWithEmptyHash(library.id).forEach {
submitTask(Task.HashBook(it, LOWEST_PRIORITY)) submitTask(Task.HashBook(it.id, LOWEST_PRIORITY, it.seriesId))
} }
} }
fun hashBookPagesWithMissingHash(library: Library) { fun hashBookPagesWithMissingHash(library: Library) {
if (library.hashPages) if (library.hashPages)
mediaRepository.findAllBookIdsByLibraryIdAndWithMissingPageHash(library.id, komgaProperties.pageHashing).forEach { mediaRepository.findAllBookIdsByLibraryIdAndWithMissingPageHash(library.id, komgaProperties.pageHashing).forEach {
submitTask(Task.HashBookPages(it, LOWEST_PRIORITY)) submitTask(Task.HashBookPages(it, LOWEST_PRIORITY, bookRepository.getSeriesIdOrNull(it) ?: ""))
} }
} }
fun convertBooksToCbz(library: Library, priority: Int = DEFAULT_PRIORITY) { fun convertBooksToCbz(library: Library, priority: Int = DEFAULT_PRIORITY) {
if (library.convertToCbz) if (library.convertToCbz)
bookConverter.getConvertibleBookIds(library).forEach { bookConverter.getConvertibleBooks(library).forEach {
submitTask(Task.ConvertBook(it, priority)) submitTask(Task.ConvertBook(it.id, priority, it.seriesId))
} }
} }
fun repairExtensions(library: Library, priority: Int = DEFAULT_PRIORITY) { fun repairExtensions(library: Library, priority: Int = DEFAULT_PRIORITY) {
if (library.repairExtensions) if (library.repairExtensions)
bookConverter.getMismatchedExtensionBookIds(library).forEach { bookConverter.getMismatchedExtensionBooks(library).forEach {
submitTask(Task.RepairExtension(it, priority)) submitTask(Task.RepairExtension(it.id, priority, it.seriesId))
} }
} }
fun analyzeBook(bookId: String, priority: Int = DEFAULT_PRIORITY) { fun analyzeBook(book: Book, priority: Int = DEFAULT_PRIORITY) {
submitTask(Task.AnalyzeBook(bookId, priority)) submitTask(Task.AnalyzeBook(book.id, priority, book.seriesId))
} }
fun generateBookThumbnail(bookId: String, priority: Int = DEFAULT_PRIORITY) { fun generateBookThumbnail(book: Book, priority: Int = DEFAULT_PRIORITY) {
submitTask(Task.GenerateBookThumbnail(bookId, priority)) submitTask(Task.GenerateBookThumbnail(book.id, priority, book.seriesId))
} }
fun refreshBookMetadata( fun refreshBookMetadata(
bookId: String, book: Book,
capabilities: Set<BookMetadataPatchCapability> = BookMetadataPatchCapability.values().toSet(), capabilities: Set<BookMetadataPatchCapability> = BookMetadataPatchCapability.values().toSet(),
priority: Int = DEFAULT_PRIORITY, priority: Int = DEFAULT_PRIORITY,
) { ) {
submitTask(Task.RefreshBookMetadata(bookId, capabilities, priority)) submitTask(Task.RefreshBookMetadata(book.id, capabilities, priority, book.seriesId))
} }
fun refreshSeriesMetadata(seriesId: String, priority: Int = DEFAULT_PRIORITY) { fun refreshSeriesMetadata(seriesId: String, priority: Int = DEFAULT_PRIORITY) {
@ -117,8 +119,8 @@ class TaskReceiver(
submitTask(Task.AggregateSeriesMetadata(seriesId, priority)) submitTask(Task.AggregateSeriesMetadata(seriesId, priority))
} }
fun refreshBookLocalArtwork(bookId: String, priority: Int = DEFAULT_PRIORITY) { fun refreshBookLocalArtwork(book: Book, priority: Int = DEFAULT_PRIORITY) {
submitTask(Task.RefreshBookLocalArtwork(bookId, priority)) submitTask(Task.RefreshBookLocalArtwork(book.id, priority, book.seriesId))
} }
fun refreshSeriesLocalArtwork(seriesId: String, priority: Int = DEFAULT_PRIORITY) { fun refreshSeriesLocalArtwork(seriesId: String, priority: Int = DEFAULT_PRIORITY) {
@ -148,6 +150,7 @@ class TaskReceiver(
setStringProperty(QUEUE_TYPE, QUEUE_TASKS_TYPE) setStringProperty(QUEUE_TYPE, QUEUE_TASKS_TYPE)
setStringProperty(QUEUE_UNIQUE_ID, task.uniqueId()) setStringProperty(QUEUE_UNIQUE_ID, task.uniqueId())
setStringProperty(QUEUE_SUB_TYPE, task::class.simpleName) setStringProperty(QUEUE_SUB_TYPE, task::class.simpleName)
task.groupId?.let { groupId -> setStringProperty("JMSXGroupID", groupId) }
} }
} }
} }

6
komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt
View file

@ -18,6 +18,9 @@ interface BookRepository {
fun findAll(bookSearch: BookSearch): Collection<Book> fun findAll(bookSearch: BookSearch): Collection<Book>
fun findAll(bookSearch: BookSearch, pageable: Pageable): Page<Book> fun findAll(bookSearch: BookSearch, pageable: Pageable): Page<Book>
fun findAllDeletedByFileSize(fileSize: Long): Collection<Book> fun findAllDeletedByFileSize(fileSize: Long): Collection<Book>
fun findAllByLibraryIdAndWithEmptyHash(libraryId: String): Collection<Book>
fun findAllByLibraryIdAndMediaTypes(libraryId: String, mediaTypes: Collection<String>): Collection<Book>
fun findAllByLibraryIdAndMismatchedExtension(libraryId: String, mediaType: String, extension: String): Collection<Book>
fun getLibraryIdOrNull(bookId: String): String? fun getLibraryIdOrNull(bookId: String): String?
fun getSeriesIdOrNull(bookId: String): String? fun getSeriesIdOrNull(bookId: String): String?
@ -28,9 +31,6 @@ interface BookRepository {
fun findAllIdsBySeriesId(seriesId: String): Collection<String> fun findAllIdsBySeriesId(seriesId: String): Collection<String>
fun findAllIdsBySeriesIds(seriesIds: Collection<String>): Collection<String> fun findAllIdsBySeriesIds(seriesIds: Collection<String>): Collection<String>
fun findAllIdsByLibraryId(libraryId: String): Collection<String> fun findAllIdsByLibraryId(libraryId: String): Collection<String>
fun findAllIdsByLibraryIdAndMediaTypes(libraryId: String, mediaTypes: Collection<String>): Collection<String>
fun findAllIdsByLibraryIdAndMismatchedExtension(libraryId: String, mediaType: String, extension: String): Collection<String>
fun findAllIdsByLibraryIdAndWithEmptyHash(libraryId: String): Collection<String>
fun findAllIds(bookSearch: BookSearch, sort: Sort): Collection<String> fun findAllIds(bookSearch: BookSearch, sort: Sort): Collection<String>
fun insert(book: Book) fun insert(book: Book)

8
komga/src/main/kotlin/org/gotson/komga/domain/service/BookConverter.kt
View file

@ -53,8 +53,8 @@ class BookConverter(
private val failedConversions = mutableListOf<String>() private val failedConversions = mutableListOf<String>()
private val skippedRepairs = mutableListOf<String>() private val skippedRepairs = mutableListOf<String>()
fun getConvertibleBookIds(library: Library): Collection<String> = fun getConvertibleBooks(library: Library): Collection<Book> =
bookRepository.findAllIdsByLibraryIdAndMediaTypes(library.id, convertibleTypes) bookRepository.findAllByLibraryIdAndMediaTypes(library.id, convertibleTypes)
fun convertToCbz(book: Book) { fun convertToCbz(book: Book) {
if (!libraryRepository.findById(book.libraryId).convertToCbz) if (!libraryRepository.findById(book.libraryId).convertToCbz)
@ -133,9 +133,9 @@ class BookConverter(
} }
} }
fun getMismatchedExtensionBookIds(library: Library): Collection<String> = fun getMismatchedExtensionBooks(library: Library): Collection<Book> =
mediaTypeToExtension.flatMap { (mediaType, extension) -> mediaTypeToExtension.flatMap { (mediaType, extension) ->
bookRepository.findAllIdsByLibraryIdAndMismatchedExtension(library.id, mediaType, extension) bookRepository.findAllByLibraryIdAndMismatchedExtension(library.id, mediaType, extension)
} }
fun repairExtension(book: Book) { fun repairExtension(book: Book) {

4
komga/src/main/kotlin/org/gotson/komga/domain/service/BookImporter.kt
View file

@ -195,8 +195,8 @@ class BookImporter(
sidecars.forEach { (sourceSidecar, destPath) -> sidecars.forEach { (sourceSidecar, destPath) ->
when (sourceSidecar.type) { when (sourceSidecar.type) {
Sidecar.Type.ARTWORK -> taskReceiver.refreshBookLocalArtwork(importedBook.id) Sidecar.Type.ARTWORK -> taskReceiver.refreshBookLocalArtwork(importedBook)
Sidecar.Type.METADATA -> taskReceiver.refreshBookMetadata(importedBook.id) Sidecar.Type.METADATA -> taskReceiver.refreshBookMetadata(importedBook)
} }
val destSidecar = sourceSidecar.copy( val destSidecar = sourceSidecar.copy(
url = destPath.toUri().toURL(), url = destPath.toUri().toURL(),

6
komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt
View file

@ -210,8 +210,8 @@ class LibraryContentLifecycle(
bookRepository.findNotDeletedByLibraryIdAndUrlOrNull(library.id, newSidecar.parentUrl)?.let { book -> bookRepository.findNotDeletedByLibraryIdAndUrlOrNull(library.id, newSidecar.parentUrl)?.let { book ->
logger.info { "Sidecar changed on disk (${newSidecar.url}, refresh Book for ${newSidecar.type}: $book" } logger.info { "Sidecar changed on disk (${newSidecar.url}, refresh Book for ${newSidecar.type}: $book" }
when (newSidecar.type) { when (newSidecar.type) {
Sidecar.Type.ARTWORK -> taskReceiver.refreshBookLocalArtwork(book.id) Sidecar.Type.ARTWORK -> taskReceiver.refreshBookLocalArtwork(book)
Sidecar.Type.METADATA -> taskReceiver.refreshBookMetadata(book.id) Sidecar.Type.METADATA -> taskReceiver.refreshBookMetadata(book)
} }
} }
} }
@ -349,7 +349,7 @@ class LibraryContentLifecycle(
title = if (deleted.titleLock) deleted.title else newlyAdded.title, title = if (deleted.titleLock) deleted.title else newlyAdded.title,
), ),
) )
if (!deleted.titleLock) taskReceiver.refreshBookMetadata(bookToAdd.id, setOf(BookMetadataPatchCapability.TITLE)) if (!deleted.titleLock) taskReceiver.refreshBookMetadata(bookToAdd, setOf(BookMetadataPatchCapability.TITLE))
} }
// copy read progress // copy read progress

17
komga/src/main/kotlin/org/gotson/komga/domain/service/SeriesLifecycle.kt
View file

@ -90,20 +90,23 @@ class SeriesLifecycle(
sorted.mapIndexed { index, (book, _) -> book.copy(number = index + 1) }, sorted.mapIndexed { index, (book, _) -> book.copy(number = index + 1) },
) )
val oldToNew = sorted.mapIndexedNotNull { index, (_, metadata) -> val oldToNew = sorted.mapIndexedNotNull { index, (book, metadata) ->
if (metadata.numberLock && metadata.numberSortLock) null if (metadata.numberLock && metadata.numberSortLock) null
else metadata to metadata.copy( else Triple(
number = if (!metadata.numberLock) (index + 1).toString() else metadata.number, book, metadata,
numberSort = if (!metadata.numberSortLock) (index + 1).toFloat() else metadata.numberSort, metadata.copy(
number = if (!metadata.numberLock) (index + 1).toString() else metadata.number,
numberSort = if (!metadata.numberSortLock) (index + 1).toFloat() else metadata.numberSort
)
) )
} }
bookMetadataRepository.update(oldToNew.map { it.second }) bookMetadataRepository.update(oldToNew.map { it.third })
// refresh metadata to reimport book number, else the series resorting would overwrite it // refresh metadata to reimport book number, else the series resorting would overwrite it
oldToNew.forEach { (old, new) -> oldToNew.forEach { (book, old, new) ->
if (old.number != new.number || old.numberSort != new.numberSort) { if (old.number != new.number || old.numberSort != new.numberSort) {
logger.debug { "Metadata numbering has changed, refreshing metadata for book ${new.bookId} " } logger.debug { "Metadata numbering has changed, refreshing metadata for book ${new.bookId} " }
taskReceiver.refreshBookMetadata(new.bookId, setOf(BookMetadataPatchCapability.NUMBER, BookMetadataPatchCapability.NUMBER_SORT)) taskReceiver.refreshBookMetadata(book, setOf(BookMetadataPatchCapability.NUMBER, BookMetadataPatchCapability.NUMBER_SORT))
} }
} }

6
komga/src/main/kotlin/org/gotson/komga/infrastructure/configuration/KomgaProperties.kt
View file

@ -46,6 +46,12 @@ class KomgaProperties {
var configDir: String? = null var configDir: String? = null
@Positive
var taskConsumers: Int = 1
@Positive
var taskConsumersMax: Int = 1
class RememberMe { class RememberMe {
@get:NotBlank @get:NotBlank
var key: String? = null var key: String? = null

22
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt
View file

@ -200,29 +200,31 @@ class BookDao(
.fetch(b.ID) .fetch(b.ID)
} }
override fun findAllIdsByLibraryIdAndMediaTypes(libraryId: String, mediaTypes: Collection<String>): Collection<String> = override fun findAllByLibraryIdAndMediaTypes(libraryId: String, mediaTypes: Collection<String>): Collection<Book> =
dsl.select(b.ID) dsl.select(*b.fields())
.from(b) .from(b)
.leftJoin(m).on(b.ID.eq(m.BOOK_ID)) .leftJoin(m).on(b.ID.eq(m.BOOK_ID))
.where(b.LIBRARY_ID.eq(libraryId)) .where(b.LIBRARY_ID.eq(libraryId))
.and(m.MEDIA_TYPE.`in`(mediaTypes)) .and(m.MEDIA_TYPE.`in`(mediaTypes))
.fetch(b.ID) .fetchInto(b)
.map { it.toDomain() }
override fun findAllIdsByLibraryIdAndMismatchedExtension(libraryId: String, mediaType: String, extension: String): Collection<String> = override fun findAllByLibraryIdAndMismatchedExtension(libraryId: String, mediaType: String, extension: String): Collection<Book> =
dsl.select(b.ID) dsl.select(*b.fields())
.from(b) .from(b)
.leftJoin(m).on(b.ID.eq(m.BOOK_ID)) .leftJoin(m).on(b.ID.eq(m.BOOK_ID))
.where(b.LIBRARY_ID.eq(libraryId)) .where(b.LIBRARY_ID.eq(libraryId))
.and(m.MEDIA_TYPE.eq(mediaType)) .and(m.MEDIA_TYPE.eq(mediaType))
.and(b.URL.notLike("%.$extension")) .and(b.URL.notLike("%.$extension"))
.fetch(b.ID) .fetchInto(b)
.map { it.toDomain() }
override fun findAllIdsByLibraryIdAndWithEmptyHash(libraryId: String): Collection<String> = override fun findAllByLibraryIdAndWithEmptyHash(libraryId: String): Collection<Book> =
dsl.select(b.ID) dsl.selectFrom(b)
.from(b)
.where(b.LIBRARY_ID.eq(libraryId)) .where(b.LIBRARY_ID.eq(libraryId))
.and(b.FILE_HASH.eq("")) .and(b.FILE_HASH.eq(""))
.fetch(b.ID) .fetchInto(b)
.map { it.toDomain() }
@Transactional @Transactional
override fun insert(book: Book) { override fun insert(book: Book) {

6
komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/BookController.kt
View file

@ -564,7 +564,7 @@ class BookController(
@ResponseStatus(HttpStatus.ACCEPTED) @ResponseStatus(HttpStatus.ACCEPTED)
fun analyze(@PathVariable bookId: String) { fun analyze(@PathVariable bookId: String) {
bookRepository.findByIdOrNull(bookId)?.let { book -> bookRepository.findByIdOrNull(bookId)?.let { book ->
taskReceiver.analyzeBook(book.id, HIGH_PRIORITY) taskReceiver.analyzeBook(book, HIGH_PRIORITY)
} ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND)
} }
@ -573,8 +573,8 @@ class BookController(
@ResponseStatus(HttpStatus.ACCEPTED) @ResponseStatus(HttpStatus.ACCEPTED)
fun refreshMetadata(@PathVariable bookId: String) { fun refreshMetadata(@PathVariable bookId: String) {
bookRepository.findByIdOrNull(bookId)?.let { book -> bookRepository.findByIdOrNull(bookId)?.let { book ->
taskReceiver.refreshBookMetadata(book.id, priority = HIGH_PRIORITY) taskReceiver.refreshBookMetadata(book, priority = HIGH_PRIORITY)
taskReceiver.refreshBookLocalArtwork(book.id, priority = HIGH_PRIORITY) taskReceiver.refreshBookLocalArtwork(book, priority = HIGH_PRIORITY)
} ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND)
} }

5
komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/LibraryController.kt
View file

@ -3,6 +3,7 @@ package org.gotson.komga.interfaces.api.rest
import org.gotson.komga.application.tasks.HIGHEST_PRIORITY import org.gotson.komga.application.tasks.HIGHEST_PRIORITY
import org.gotson.komga.application.tasks.HIGH_PRIORITY import org.gotson.komga.application.tasks.HIGH_PRIORITY
import org.gotson.komga.application.tasks.TaskReceiver import org.gotson.komga.application.tasks.TaskReceiver
import org.gotson.komga.domain.model.BookSearch
import org.gotson.komga.domain.model.DirectoryNotFoundException import org.gotson.komga.domain.model.DirectoryNotFoundException
import org.gotson.komga.domain.model.DuplicateNameException import org.gotson.komga.domain.model.DuplicateNameException
import org.gotson.komga.domain.model.Library import org.gotson.komga.domain.model.Library
@ -178,7 +179,7 @@ class LibraryController(
@PreAuthorize("hasRole('$ROLE_ADMIN')") @PreAuthorize("hasRole('$ROLE_ADMIN')")
@ResponseStatus(HttpStatus.ACCEPTED) @ResponseStatus(HttpStatus.ACCEPTED)
fun analyze(@PathVariable libraryId: String) { fun analyze(@PathVariable libraryId: String) {
bookRepository.findAllIdsByLibraryId(libraryId).forEach { bookRepository.findAll(BookSearch(libraryIds = listOf(libraryId))).forEach {
taskReceiver.analyzeBook(it, HIGH_PRIORITY) taskReceiver.analyzeBook(it, HIGH_PRIORITY)
} }
} }
@ -187,7 +188,7 @@ class LibraryController(
@PreAuthorize("hasRole('$ROLE_ADMIN')") @PreAuthorize("hasRole('$ROLE_ADMIN')")
@ResponseStatus(HttpStatus.ACCEPTED) @ResponseStatus(HttpStatus.ACCEPTED)
fun refreshMetadata(@PathVariable libraryId: String) { fun refreshMetadata(@PathVariable libraryId: String) {
bookRepository.findAllIdsByLibraryId(libraryId).forEach { bookRepository.findAll(BookSearch(libraryIds = listOf(libraryId))).forEach {
taskReceiver.refreshBookMetadata(it, priority = HIGH_PRIORITY) taskReceiver.refreshBookMetadata(it, priority = HIGH_PRIORITY)
taskReceiver.refreshBookLocalArtwork(it, priority = HIGH_PRIORITY) taskReceiver.refreshBookLocalArtwork(it, priority = HIGH_PRIORITY)
} }

4
komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/SeriesController.kt
View file

@ -488,7 +488,7 @@ class SeriesController(
@PreAuthorize("hasRole('$ROLE_ADMIN')") @PreAuthorize("hasRole('$ROLE_ADMIN')")
@ResponseStatus(HttpStatus.ACCEPTED) @ResponseStatus(HttpStatus.ACCEPTED)
fun analyze(@PathVariable seriesId: String) { fun analyze(@PathVariable seriesId: String) {
bookRepository.findAllIdsBySeriesId(seriesId).forEach { bookRepository.findAllBySeriesId(seriesId).forEach {
taskReceiver.analyzeBook(it, HIGH_PRIORITY) taskReceiver.analyzeBook(it, HIGH_PRIORITY)
} }
} }
@ -497,7 +497,7 @@ class SeriesController(
@PreAuthorize("hasRole('$ROLE_ADMIN')") @PreAuthorize("hasRole('$ROLE_ADMIN')")
@ResponseStatus(HttpStatus.ACCEPTED) @ResponseStatus(HttpStatus.ACCEPTED)
fun refreshMetadata(@PathVariable seriesId: String) { fun refreshMetadata(@PathVariable seriesId: String) {
bookRepository.findAllIdsBySeriesId(seriesId).forEach { bookRepository.findAllBySeriesId(seriesId).forEach {
taskReceiver.refreshBookMetadata(it, priority = HIGH_PRIORITY) taskReceiver.refreshBookMetadata(it, priority = HIGH_PRIORITY)
taskReceiver.refreshBookLocalArtwork(it, priority = HIGH_PRIORITY) taskReceiver.refreshBookLocalArtwork(it, priority = HIGH_PRIORITY)
} }

2
komga/src/main/resources/application-dev.yml
View file

@ -9,6 +9,8 @@ komga:
file: ":memory:" file: ":memory:"
cors.allowed-origins: cors.allowed-origins:
- http://localhost:8081 - http://localhost:8081
task-consumers: 5
task-consumers-max: 20
# delete-empty-collections: true # delete-empty-collections: true
# delete-empty-read-lists: true # delete-empty-read-lists: true
oauth2-account-creation: false oauth2-account-creation: false

5
komga/src/test/kotlin/org/gotson/komga/application/tasks/TaskHandlerTest.kt
View file

@ -66,8 +66,9 @@ class TaskHandlerTest(
every { mockBookLifecycle.analyzeAndPersist(any()) } returns false every { mockBookLifecycle.analyzeAndPersist(any()) } returns false
jmsListenerEndpointRegistry.stop() jmsListenerEndpointRegistry.stop()
val book = makeBook("book")
repeat(100) { repeat(100) {
taskReceiver.analyzeBook("id") taskReceiver.analyzeBook(book)
} }
jmsListenerEndpointRegistry.start() jmsListenerEndpointRegistry.start()
@ -88,7 +89,7 @@ class TaskHandlerTest(
jmsListenerEndpointRegistry.stop() jmsListenerEndpointRegistry.stop()
(0..9).forEach { (0..9).forEach {
taskReceiver.analyzeBook("$it", it) taskReceiver.analyzeBook(makeBook("$it", id = "$it"), it)
} }
jmsListenerEndpointRegistry.start() jmsListenerEndpointRegistry.start()

10
komga/src/test/kotlin/org/gotson/komga/domain/model/Utils.kt
View file

@ -4,7 +4,14 @@ import com.github.f4b6a3.tsid.TsidCreator
import java.net.URL import java.net.URL
import java.time.LocalDateTime import java.time.LocalDateTime
fun makeBook(name: String, fileLastModified: LocalDateTime = LocalDateTime.now(), libraryId: String = "", seriesId: String = "", url: URL? = null): Book { fun makeBook(
name: String,
fileLastModified: LocalDateTime = LocalDateTime.now(),
libraryId: String = "",
seriesId: String = "",
url: URL? = null,
id: String = TsidCreator.getTsid256().toString(),
): Book {
Thread.sleep(5) Thread.sleep(5)
return Book( return Book(
name = name, name = name,
@ -12,6 +19,7 @@ fun makeBook(name: String, fileLastModified: LocalDateTime = LocalDateTime.now()
fileLastModified = fileLastModified, fileLastModified = fileLastModified,
libraryId = libraryId, libraryId = libraryId,
seriesId = seriesId, seriesId = seriesId,
id = id,
) )
} }

8
komga/src/test/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycleTest.kt
View file

@ -817,7 +817,7 @@ class LibraryContentLifecycleTest(
// then // then
verify(exactly = 1) { mockHasher.computeHash(any<Path>()) } verify(exactly = 1) { mockHasher.computeHash(any<Path>()) }
verify(exactly = 0) { mockTaskReceiver.refreshBookMetadata(bookRenamed.id, setOf(BookMetadataPatchCapability.TITLE)) } verify(exactly = 0) { mockTaskReceiver.refreshBookMetadata(bookRenamed, setOf(BookMetadataPatchCapability.TITLE)) }
val allSeries = seriesRepository.findAll() val allSeries = seriesRepository.findAll()
val allBooks = bookRepository.findAll().sortedBy { it.number } val allBooks = bookRepository.findAll().sortedBy { it.number }
@ -861,7 +861,7 @@ class LibraryContentLifecycleTest(
// then // then
verify(exactly = 1) { mockHasher.computeHash(any<Path>()) } verify(exactly = 1) { mockHasher.computeHash(any<Path>()) }
verify(exactly = 1) { mockTaskReceiver.refreshBookMetadata(bookRenamed.id, setOf(BookMetadataPatchCapability.TITLE)) } verify(exactly = 1) { mockTaskReceiver.refreshBookMetadata(withArg { assertThat(it.id).isEqualTo(bookRenamed.id) }, setOf(BookMetadataPatchCapability.TITLE)) }
val allSeries = seriesRepository.findAll() val allSeries = seriesRepository.findAll()
val allBooks = bookRepository.findAll().sortedBy { it.number } val allBooks = bookRepository.findAll().sortedBy { it.number }
@ -1186,7 +1186,7 @@ class LibraryContentLifecycleTest(
libraryContentLifecycle.scanRootFolder(library) // rename libraryContentLifecycle.scanRootFolder(library) // rename
// then // then
verify(exactly = 0) { mockTaskReceiver.refreshBookMetadata(book2Moved.id, setOf(BookMetadataPatchCapability.TITLE)) } verify(exactly = 0) { mockTaskReceiver.refreshBookMetadata(book2Moved, setOf(BookMetadataPatchCapability.TITLE)) }
val allSeries = seriesRepository.findAll() val allSeries = seriesRepository.findAll()
val allBooks = bookRepository.findAll().sortedBy { it.number } val allBooks = bookRepository.findAll().sortedBy { it.number }
@ -1244,7 +1244,7 @@ class LibraryContentLifecycleTest(
libraryContentLifecycle.scanRootFolder(library) // rename libraryContentLifecycle.scanRootFolder(library) // rename
// then // then
verify(exactly = 1) { mockTaskReceiver.refreshBookMetadata(book2Moved.id, setOf(BookMetadataPatchCapability.TITLE)) } verify(exactly = 1) { mockTaskReceiver.refreshBookMetadata(withArg { assertThat(it.id).isEqualTo(book2Moved.id) }, setOf(BookMetadataPatchCapability.TITLE)) }
val allSeries = seriesRepository.findAll() val allSeries = seriesRepository.findAll()
val allBooks = bookRepository.findAll().sortedBy { it.number } val allBooks = bookRepository.findAll().sortedBy { it.number }