feat(api): search series by completeness

This commit is contained in:
Gauthier Roebroeck 2021-12-31 10:16:54 +08:00
parent e4b912e607
commit 494bdf28a1
10 changed files with 35 additions and 17 deletions

View file

@ -2,6 +2,7 @@ package org.gotson.komga.application.tasks
import org.gotson.komga.domain.model.BookMetadataPatchCapability import org.gotson.komga.domain.model.BookMetadataPatchCapability
import org.gotson.komga.domain.model.CopyMode import org.gotson.komga.domain.model.CopyMode
import org.gotson.komga.infrastructure.search.LuceneEntity
import java.io.Serializable import java.io.Serializable
const val HIGHEST_PRIORITY = 8 const val HIGHEST_PRIORITY = 8
@ -79,9 +80,9 @@ sealed class Task(priority: Int = DEFAULT_PRIORITY) : Serializable {
override fun toString(): String = "RepairExtension(bookId='$bookId', priority='$priority')" override fun toString(): String = "RepairExtension(bookId='$bookId', priority='$priority')"
} }
class RebuildIndex(priority: Int = DEFAULT_PRIORITY) : Task(priority) { class RebuildIndex(val entities: Set<LuceneEntity>?, priority: Int = DEFAULT_PRIORITY) : Task(priority) {
override fun uniqueId() = "REBUILD_INDEX" override fun uniqueId() = "REBUILD_INDEX"
override fun toString(): String = "RebuildIndex(priority='$priority')" override fun toString(): String = "RebuildIndex(priority='$priority',entities='${entities?.map { it.type }}')"
} }
class DeleteBook(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) { class DeleteBook(val bookId: String, priority: Int = DEFAULT_PRIORITY) : Task(priority) {

View file

@ -122,7 +122,7 @@ class TaskHandler(
bookLifecycle.hashAndPersist(book) bookLifecycle.hashAndPersist(book)
} ?: logger.warn { "Cannot execute task $task: Book does not exist" } } ?: logger.warn { "Cannot execute task $task: Book does not exist" }
is Task.RebuildIndex -> searchIndexLifecycle.rebuildIndex() is Task.RebuildIndex -> searchIndexLifecycle.rebuildIndex(task.entities)
is Task.DeleteBook -> { is Task.DeleteBook -> {
bookRepository.findByIdOrNull(task.bookId)?.let { book -> bookRepository.findByIdOrNull(task.bookId)?.let { book ->

View file

@ -15,6 +15,7 @@ import org.gotson.komga.infrastructure.jms.QUEUE_TASKS
import org.gotson.komga.infrastructure.jms.QUEUE_TASKS_TYPE import org.gotson.komga.infrastructure.jms.QUEUE_TASKS_TYPE
import org.gotson.komga.infrastructure.jms.QUEUE_TYPE import org.gotson.komga.infrastructure.jms.QUEUE_TYPE
import org.gotson.komga.infrastructure.jms.QUEUE_UNIQUE_ID import org.gotson.komga.infrastructure.jms.QUEUE_UNIQUE_ID
import org.gotson.komga.infrastructure.search.LuceneEntity
import org.springframework.data.domain.Sort import org.springframework.data.domain.Sort
import org.springframework.jms.core.JmsTemplate import org.springframework.jms.core.JmsTemplate
import org.springframework.stereotype.Service import org.springframework.stereotype.Service
@ -117,8 +118,8 @@ class TaskReceiver(
submitTask(Task.ImportBook(sourceFile, seriesId, copyMode, destinationName, upgradeBookId, priority)) submitTask(Task.ImportBook(sourceFile, seriesId, copyMode, destinationName, upgradeBookId, priority))
} }
fun rebuildIndex(priority: Int = DEFAULT_PRIORITY) { fun rebuildIndex(priority: Int = DEFAULT_PRIORITY, entities: Set<LuceneEntity>? = null) {
submitTask(Task.RebuildIndex(priority)) submitTask(Task.RebuildIndex(entities, priority))
} }
fun deleteBook(bookId: String, priority: Int = DEFAULT_PRIORITY) { fun deleteBook(bookId: String, priority: Int = DEFAULT_PRIORITY) {

View file

@ -8,6 +8,7 @@ open class SeriesSearch(
val metadataStatus: Collection<SeriesMetadata.Status>? = null, val metadataStatus: Collection<SeriesMetadata.Status>? = null,
val publishers: Collection<String>? = null, val publishers: Collection<String>? = null,
val deleted: Boolean? = null, val deleted: Boolean? = null,
val complete: Boolean? = null,
) { ) {
enum class SearchField { enum class SearchField {
NAME, TITLE, TITLE_SORT NAME, TITLE, TITLE_SORT
@ -22,6 +23,7 @@ class SeriesSearchWithReadProgress(
metadataStatus: Collection<SeriesMetadata.Status>? = null, metadataStatus: Collection<SeriesMetadata.Status>? = null,
publishers: Collection<String>? = null, publishers: Collection<String>? = null,
deleted: Boolean? = null, deleted: Boolean? = null,
complete: Boolean? = null,
val languages: Collection<String>? = null, val languages: Collection<String>? = null,
val genres: Collection<String>? = null, val genres: Collection<String>? = null,
val tags: Collection<String>? = null, val tags: Collection<String>? = null,
@ -37,4 +39,5 @@ class SeriesSearchWithReadProgress(
metadataStatus = metadataStatus, metadataStatus = metadataStatus,
publishers = publishers, publishers = publishers,
deleted = deleted, deleted = deleted,
complete = complete,
) )

View file

@ -87,7 +87,7 @@ class SeriesDtoDao(
collectionId: String, collectionId: String,
search: SeriesSearchWithReadProgress, search: SeriesSearchWithReadProgress,
userId: String, userId: String,
pageable: Pageable pageable: Pageable,
): Page<SeriesDto> { ): Page<SeriesDto> {
val conditions = search.toCondition().and(cs.COLLECTION_ID.eq(collectionId)) val conditions = search.toCondition().and(cs.COLLECTION_ID.eq(collectionId))
val joinConditions = search.toJoinConditions().copy(selectCollectionNumber = true, collection = true) val joinConditions = search.toJoinConditions().copy(selectCollectionNumber = true, collection = true)
@ -98,7 +98,7 @@ class SeriesDtoDao(
override fun findAllRecentlyUpdated( override fun findAllRecentlyUpdated(
search: SeriesSearchWithReadProgress, search: SeriesSearchWithReadProgress,
userId: String, userId: String,
pageable: Pageable pageable: Pageable,
): Page<SeriesDto> { ): Page<SeriesDto> {
val conditions = search.toCondition() val conditions = search.toCondition()
.and(s.CREATED_DATE.ne(s.LAST_MODIFIED_DATE)) .and(s.CREATED_DATE.ne(s.LAST_MODIFIED_DATE))
@ -143,7 +143,7 @@ class SeriesDtoDao(
private fun selectBase( private fun selectBase(
userId: String, userId: String,
joinConditions: JoinConditions = JoinConditions() joinConditions: JoinConditions = JoinConditions(),
): SelectOnConditionStep<Record> = ): SelectOnConditionStep<Record> =
dsl.selectDistinct(*groupFields) dsl.selectDistinct(*groupFields)
.apply { if (joinConditions.selectCollectionNumber) select(cs.NUMBER) } .apply { if (joinConditions.selectCollectionNumber) select(cs.NUMBER) }
@ -205,7 +205,7 @@ class SeriesDtoDao(
dtos, dtos,
if (pageable.isPaged) PageRequest.of(pageable.pageNumber, pageable.pageSize, pageSort) if (pageable.isPaged) PageRequest.of(pageable.pageNumber, pageable.pageSize, pageSort)
else PageRequest.of(0, maxOf(count, 20), pageSort), else PageRequest.of(0, maxOf(count, 20), pageSort),
count.toLong() count.toLong(),
) )
} }
@ -248,7 +248,7 @@ class SeriesDtoDao(
booksUnreadCount, booksUnreadCount,
booksInProgressCount, booksInProgressCount,
dr.toDto(genres, tags), dr.toDto(genres, tags),
bmar.toDto(aggregatedAuthors, aggregatedTags) bmar.toDto(aggregatedAuthors, aggregatedTags),
) )
} }
@ -262,6 +262,8 @@ class SeriesDtoDao(
if (!publishers.isNullOrEmpty()) c = c.and(lower(d.PUBLISHER).`in`(publishers.map { it.lowercase() })) if (!publishers.isNullOrEmpty()) c = c.and(lower(d.PUBLISHER).`in`(publishers.map { it.lowercase() }))
if (deleted == true) c = c.and(s.DELETED_DATE.isNotNull) if (deleted == true) c = c.and(s.DELETED_DATE.isNotNull)
if (deleted == false) c = c.and(s.DELETED_DATE.isNull) if (deleted == false) c = c.and(s.DELETED_DATE.isNull)
if (complete == false) c = c.and(d.TOTAL_BOOK_COUNT.isNotNull.and(d.TOTAL_BOOK_COUNT.ne(s.BOOK_COUNT)))
if (complete == true) c = c.and(d.TOTAL_BOOK_COUNT.isNotNull.and(d.TOTAL_BOOK_COUNT.eq(s.BOOK_COUNT)))
if (!languages.isNullOrEmpty()) c = c.and(lower(d.LANGUAGE).`in`(languages.map { it.lowercase() })) if (!languages.isNullOrEmpty()) c = c.and(lower(d.LANGUAGE).`in`(languages.map { it.lowercase() }))
if (!genres.isNullOrEmpty()) c = c.and(lower(g.GENRE).`in`(genres.map { it.lowercase() })) if (!genres.isNullOrEmpty()) c = c.and(lower(g.GENRE).`in`(genres.map { it.lowercase() }))
if (!tags.isNullOrEmpty()) c = c.and(lower(st.TAG).`in`(tags.map { it.lowercase() }).or(lower(bmat.TAG).`in`(tags.map { it.lowercase() }))) if (!tags.isNullOrEmpty()) c = c.and(lower(st.TAG).`in`(tags.map { it.lowercase() }).or(lower(bmat.TAG).`in`(tags.map { it.lowercase() })))
@ -322,7 +324,7 @@ class SeriesDtoDao(
booksUnreadCount: Int, booksUnreadCount: Int,
booksInProgressCount: Int, booksInProgressCount: Int,
metadata: SeriesMetadataDto, metadata: SeriesMetadataDto,
booksMetadata: BookMetadataAggregationDto booksMetadata: BookMetadataAggregationDto,
) = ) =
SeriesDto( SeriesDto(
id = id, id = id,
@ -378,6 +380,6 @@ class SeriesDtoDao(
summaryNumber = summaryNumber, summaryNumber = summaryNumber,
created = createdDate.toCurrentTimeZone(), created = createdDate.toCurrentTimeZone(),
lastModified = lastModifiedDate.toCurrentTimeZone() lastModified = lastModifiedDate.toCurrentTimeZone(),
) )
} }

View file

@ -69,6 +69,7 @@ fun SeriesDto.toDocument() =
} }
if (booksMetadata.releaseDate != null) add(TextField("release_date", DateTools.dateToString(booksMetadata.releaseDate.toDate(), DateTools.Resolution.YEAR), Field.Store.NO)) if (booksMetadata.releaseDate != null) add(TextField("release_date", DateTools.dateToString(booksMetadata.releaseDate.toDate(), DateTools.Resolution.YEAR), Field.Store.NO))
add(TextField("deleted", deleted.toString(), Field.Store.NO)) add(TextField("deleted", deleted.toString(), Field.Store.NO))
if (metadata.totalBookCount != null) add(TextField("complete", (metadata.totalBookCount == booksCount).toString(), Field.Store.NO))
add(StringField(LuceneEntity.TYPE, LuceneEntity.Series.type, Field.Store.NO)) add(StringField(LuceneEntity.TYPE, LuceneEntity.Series.type, Field.Store.NO))
add(StringField(LuceneEntity.Series.id, id, Field.Store.YES)) add(StringField(LuceneEntity.Series.id, id, Field.Store.YES))

View file

@ -26,7 +26,7 @@ import kotlin.math.ceil
import kotlin.time.measureTime import kotlin.time.measureTime
private val logger = KotlinLogging.logger {} private val logger = KotlinLogging.logger {}
private const val INDEX_VERSION = 3 private const val INDEX_VERSION = 4
@Component @Component
class SearchIndexLifecycle( class SearchIndexLifecycle(
@ -37,10 +37,12 @@ class SearchIndexLifecycle(
private val luceneHelper: LuceneHelper, private val luceneHelper: LuceneHelper,
) { ) {
fun rebuildIndex() { fun rebuildIndex(entities: Set<LuceneEntity>?) {
logger.info { "Rebuild all indexes" } val targetEntities = entities ?: LuceneEntity.values().toSet()
LuceneEntity.values().forEach { logger.info { "Rebuild index for: ${targetEntities.map { it.type }}" }
targetEntities.forEach {
when (it) { when (it) {
LuceneEntity.Book -> rebuildIndex(it, { p: Pageable -> bookDtoRepository.findAll(BookSearchWithReadProgress(), "unused", p) }, { e: BookDto -> e.toDocument() }) LuceneEntity.Book -> rebuildIndex(it, { p: Pageable -> bookDtoRepository.findAll(BookSearchWithReadProgress(), "unused", p) }, { e: BookDto -> e.toDocument() })
LuceneEntity.Series -> rebuildIndex(it, { p: Pageable -> seriesDtoRepository.findAll(SeriesSearchWithReadProgress(), "unused", p) }, { e: SeriesDto -> e.toDocument() }) LuceneEntity.Series -> rebuildIndex(it, { p: Pageable -> seriesDtoRepository.findAll(SeriesSearchWithReadProgress(), "unused", p) }, { e: SeriesDto -> e.toDocument() })
@ -63,7 +65,7 @@ class SearchIndexLifecycle(
indexWriter.deleteDocuments(Term(LuceneEntity.TYPE, entity.type)) indexWriter.deleteDocuments(Term(LuceneEntity.TYPE, entity.type))
(0 until pages).forEach { page -> (0 until pages).forEach { page ->
logger.info { "Processing page $page of $batchSize elements" } logger.info { "Processing page ${page + 1} of $pages ($batchSize elements)" }
val entityDocs = provider(PageRequest.of(page, batchSize)).content val entityDocs = provider(PageRequest.of(page, batchSize)).content
.map { toDoc(it) } .map { toDoc(it) }
indexWriter.addDocuments(entityDocs) indexWriter.addDocuments(entityDocs)

View file

@ -263,6 +263,7 @@ class SeriesCollectionController(
@RequestParam(name = "age_rating", required = false) ageRatings: List<String>?, @RequestParam(name = "age_rating", required = false) ageRatings: List<String>?,
@RequestParam(name = "release_year", required = false) release_years: List<String>?, @RequestParam(name = "release_year", required = false) release_years: List<String>?,
@RequestParam(name = "deleted", required = false) deleted: Boolean?, @RequestParam(name = "deleted", required = false) deleted: Boolean?,
@RequestParam(name = "complete", required = false) complete: Boolean?,
@RequestParam(name = "unpaged", required = false) unpaged: Boolean = false, @RequestParam(name = "unpaged", required = false) unpaged: Boolean = false,
@Parameter(hidden = true) @Authors authors: List<Author>?, @Parameter(hidden = true) @Authors authors: List<Author>?,
@Parameter(hidden = true) page: Pageable @Parameter(hidden = true) page: Pageable
@ -286,6 +287,7 @@ class SeriesCollectionController(
readStatus = readStatus, readStatus = readStatus,
publishers = publishers, publishers = publishers,
deleted = deleted, deleted = deleted,
complete = complete,
languages = languages, languages = languages,
genres = genres, genres = genres,
tags = tags, tags = tags,

View file

@ -131,6 +131,7 @@ class SeriesController(
@RequestParam(name = "age_rating", required = false) ageRatings: List<String>?, @RequestParam(name = "age_rating", required = false) ageRatings: List<String>?,
@RequestParam(name = "release_year", required = false) release_years: List<String>?, @RequestParam(name = "release_year", required = false) release_years: List<String>?,
@RequestParam(name = "deleted", required = false) deleted: Boolean?, @RequestParam(name = "deleted", required = false) deleted: Boolean?,
@RequestParam(name = "complete", required = false) complete: Boolean?,
@RequestParam(name = "unpaged", required = false) unpaged: Boolean = false, @RequestParam(name = "unpaged", required = false) unpaged: Boolean = false,
@Parameter(hidden = true) @Authors authors: List<Author>?, @Parameter(hidden = true) @Authors authors: List<Author>?,
@Parameter(hidden = true) page: Pageable @Parameter(hidden = true) page: Pageable
@ -165,6 +166,7 @@ class SeriesController(
readStatus = readStatus, readStatus = readStatus,
publishers = publishers, publishers = publishers,
deleted = deleted, deleted = deleted,
complete = complete,
languages = languages, languages = languages,
genres = genres, genres = genres,
tags = tags, tags = tags,
@ -200,6 +202,7 @@ class SeriesController(
@RequestParam(name = "age_rating", required = false) ageRatings: List<String>?, @RequestParam(name = "age_rating", required = false) ageRatings: List<String>?,
@RequestParam(name = "release_year", required = false) release_years: List<String>?, @RequestParam(name = "release_year", required = false) release_years: List<String>?,
@RequestParam(name = "deleted", required = false) deleted: Boolean?, @RequestParam(name = "deleted", required = false) deleted: Boolean?,
@RequestParam(name = "complete", required = false) complete: Boolean?,
@Parameter(hidden = true) @Authors authors: List<Author>?, @Parameter(hidden = true) @Authors authors: List<Author>?,
@Parameter(hidden = true) page: Pageable @Parameter(hidden = true) page: Pageable
): List<GroupCountDto> { ): List<GroupCountDto> {
@ -218,6 +221,7 @@ class SeriesController(
readStatus = readStatus, readStatus = readStatus,
publishers = publishers, publishers = publishers,
deleted = deleted, deleted = deleted,
complete = complete,
languages = languages, languages = languages,
genres = genres, genres = genres,
tags = tags, tags = tags,

View file

@ -3,6 +3,7 @@ package org.gotson.komga.interfaces.scheduler
import mu.KotlinLogging import mu.KotlinLogging
import org.gotson.komga.application.tasks.HIGHEST_PRIORITY import org.gotson.komga.application.tasks.HIGHEST_PRIORITY
import org.gotson.komga.application.tasks.TaskReceiver import org.gotson.komga.application.tasks.TaskReceiver
import org.gotson.komga.infrastructure.search.LuceneEntity
import org.gotson.komga.infrastructure.search.LuceneHelper import org.gotson.komga.infrastructure.search.LuceneHelper
import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.boot.context.event.ApplicationReadyEvent
import org.springframework.context.annotation.Profile import org.springframework.context.annotation.Profile
@ -27,6 +28,7 @@ class SearchIndexController(
logger.info { "Lucene index version: ${luceneHelper.getIndexVersion()}" } logger.info { "Lucene index version: ${luceneHelper.getIndexVersion()}" }
when (luceneHelper.getIndexVersion()) { when (luceneHelper.getIndexVersion()) {
1, 2 -> taskReceiver.rebuildIndex(HIGHEST_PRIORITY) 1, 2 -> taskReceiver.rebuildIndex(HIGHEST_PRIORITY)
3 -> taskReceiver.rebuildIndex(HIGHEST_PRIORITY, setOf(LuceneEntity.Series))
} }
} }
} }