From 30c349afaffdcd307a422653895e4fe91dc47f29 Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Mon, 26 Jul 2021 10:09:06 +0800 Subject: [PATCH] fix(api): accent insensitive search --- .../datasource/SqliteUdfDataSource.kt | 28 +++++++++++++++++-- .../komga/infrastructure/jooq/BookDtoDao.kt | 4 ++- .../komga/infrastructure/jooq/ReadListDao.kt | 7 +++-- .../infrastructure/jooq/ReferentialDao.kt | 13 +++++---- .../jooq/SeriesCollectionDao.kt | 7 +++-- .../komga/infrastructure/jooq/SeriesDtoDao.kt | 3 +- .../gotson/komga/infrastructure/jooq/Utils.kt | 6 ++++ .../komga/infrastructure/language/Utils.kt | 3 ++ 8 files changed, 55 insertions(+), 16 deletions(-) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/datasource/SqliteUdfDataSource.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/datasource/SqliteUdfDataSource.kt index 3ce2b7e7b..0af43a14c 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/datasource/SqliteUdfDataSource.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/datasource/SqliteUdfDataSource.kt @@ -1,6 +1,7 @@ package org.gotson.komga.infrastructure.datasource import mu.KotlinLogging +import org.gotson.komga.infrastructure.language.stripAccents import org.springframework.jdbc.datasource.SimpleDriverDataSource import org.sqlite.Function import org.sqlite.SQLiteConnection @@ -10,11 +11,20 @@ private val log = KotlinLogging.logger {} class SqliteUdfDataSource : SimpleDriverDataSource() { + companion object { + const val udfStripAccents = "UDF_UNIDECODE" + } + override fun getConnection(): Connection = - super.getConnection().also { createUdfRegexp(it as SQLiteConnection) } + super.getConnection().also { addAllUdf(it as SQLiteConnection) } override fun getConnection(username: String, password: String): Connection = - super.getConnection(username, password).also { createUdfRegexp(it as SQLiteConnection) } + super.getConnection(username, password).also { addAllUdf(it as SQLiteConnection) } + + private fun addAllUdf(connection: SQLiteConnection) { + createUdfRegexp(connection) + createUdfStripAccents(connection) + } private fun createUdfRegexp(connection: SQLiteConnection) { log.debug { "Adding custom REGEXP function" } @@ -30,4 +40,18 @@ class SqliteUdfDataSource : SimpleDriverDataSource() { } ) } + + private fun createUdfStripAccents(connection: SQLiteConnection) { + log.debug { "Adding custom $udfStripAccents function" } + Function.create( + connection, udfStripAccents, + object : Function() { + override fun xFunc() = + when (val text = value_text(0)) { + null -> error("Argument must not be null") + else -> result(text.stripAccents()) + } + } + ) + } } diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt index da53fc4d8..a05485a1c 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt @@ -2,6 +2,7 @@ package org.gotson.komga.infrastructure.jooq import org.gotson.komga.domain.model.BookSearchWithReadProgress import org.gotson.komga.domain.model.ReadStatus +import org.gotson.komga.infrastructure.language.stripAccents import org.gotson.komga.infrastructure.web.toFilePath import org.gotson.komga.interfaces.rest.dto.AuthorDto import org.gotson.komga.interfaces.rest.dto.BookDto @@ -56,6 +57,7 @@ class BookDtoDao( "media.comment" to lower(m.COMMENT), "media.mediaType" to lower(m.MEDIA_TYPE), "metadata.numberSort" to d.NUMBER_SORT, + "metadata.title" to lower(d.TITLE), "metadata.releaseDate" to d.RELEASE_DATE, "readProgress.lastModified" to r.LAST_MODIFIED_DATE, "readList.number" to rlb.NUMBER @@ -266,7 +268,7 @@ class BookDtoDao( if (!libraryIds.isNullOrEmpty()) c = c.and(b.LIBRARY_ID.`in`(libraryIds)) if (!seriesIds.isNullOrEmpty()) c = c.and(b.SERIES_ID.`in`(seriesIds)) - searchTerm?.let { c = c.and(d.TITLE.containsIgnoreCase(it)) } + searchTerm?.let { c = c.and(d.TITLE.udfStripAccents().containsIgnoreCase(it.stripAccents())) } if (!mediaStatus.isNullOrEmpty()) c = c.and(m.STATUS.`in`(mediaStatus)) if (deleted == true) c = c.and(b.DELETED_DATE.isNotNull) if (deleted == false) c = c.and(b.DELETED_DATE.isNull) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt index bb57cf888..3255afb43 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt @@ -2,6 +2,7 @@ package org.gotson.komga.infrastructure.jooq import org.gotson.komga.domain.model.ReadList import org.gotson.komga.domain.persistence.ReadListRepository +import org.gotson.komga.infrastructure.language.stripAccents import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.ReadlistRecord import org.jooq.DSLContext @@ -46,7 +47,7 @@ class ReadListDao( .firstOrNull() override fun searchAll(search: String?, pageable: Pageable): Page { - val conditions = search?.let { rl.NAME.containsIgnoreCase(it) } + val conditions = search?.let { rl.NAME.udfStripAccents().containsIgnoreCase(it.stripAccents()) } ?: DSL.trueCondition() val count = dsl.selectCount() @@ -77,7 +78,7 @@ class ReadListDao( .leftJoin(rlb).on(rl.ID.eq(rlb.READLIST_ID)) .leftJoin(b).on(rlb.BOOK_ID.eq(b.ID)) .where(b.LIBRARY_ID.`in`(belongsToLibraryIds)) - .apply { search?.let { and(rl.NAME.containsIgnoreCase(it)) } } + .apply { search?.let { and(rl.NAME.udfStripAccents().containsIgnoreCase(it.stripAccents())) } } .fetch(0, String::class.java) val count = ids.size @@ -87,7 +88,7 @@ class ReadListDao( val items = selectBase() .where(rl.ID.`in`(ids)) .apply { filterOnLibraryIds?.let { and(b.LIBRARY_ID.`in`(it)) } } - .apply { search?.let { and(rl.NAME.containsIgnoreCase(it)) } } + .apply { search?.let { and(rl.NAME.udfStripAccents().containsIgnoreCase(it.stripAccents())) } } .orderBy(orderBy) .apply { if (pageable.isPaged) limit(pageable.pageSize).offset(pageable.offset) } .fetchAndMap(filterOnLibraryIds) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReferentialDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReferentialDao.kt index 8ee8d60d9..1b84571f8 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReferentialDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReferentialDao.kt @@ -2,6 +2,7 @@ package org.gotson.komga.infrastructure.jooq import org.gotson.komga.domain.model.Author import org.gotson.komga.domain.persistence.ReferentialRepository +import org.gotson.komga.infrastructure.language.stripAccents import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.BookMetadataAggregationAuthorRecord import org.gotson.komga.jooq.tables.records.BookMetadataAuthorRecord @@ -36,7 +37,7 @@ class ReferentialDao( dsl.selectDistinct(a.NAME, a.ROLE) .from(a) .apply { filterOnLibraryIds?.let { leftJoin(b).on(a.BOOK_ID.eq(b.ID)) } } - .where(a.NAME.containsIgnoreCase(search)) + .where(a.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) .apply { filterOnLibraryIds?.let { and(b.LIBRARY_ID.`in`(it)) } } .orderBy(lower(a.NAME), a.ROLE) .fetchInto(a) @@ -46,7 +47,7 @@ class ReferentialDao( dsl.selectDistinct(bmaa.NAME, bmaa.ROLE) .from(bmaa) .leftJoin(s).on(bmaa.SERIES_ID.eq(s.ID)) - .where(bmaa.NAME.containsIgnoreCase(search)) + .where(bmaa.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) .and(s.LIBRARY_ID.eq(libraryId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } .orderBy(lower(bmaa.NAME), bmaa.ROLE) @@ -58,7 +59,7 @@ class ReferentialDao( .from(bmaa) .leftJoin(cs).on(bmaa.SERIES_ID.eq(cs.SERIES_ID)) .apply { filterOnLibraryIds?.let { leftJoin(s).on(bmaa.SERIES_ID.eq(s.ID)) } } - .where(bmaa.NAME.containsIgnoreCase(search)) + .where(bmaa.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) .and(cs.COLLECTION_ID.eq(collectionId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } .orderBy(lower(bmaa.NAME), bmaa.ROLE) @@ -69,7 +70,7 @@ class ReferentialDao( dsl.selectDistinct(bmaa.NAME, bmaa.ROLE) .from(bmaa) .apply { filterOnLibraryIds?.let { leftJoin(s).on(bmaa.SERIES_ID.eq(s.ID)) } } - .where(bmaa.NAME.containsIgnoreCase(search)) + .where(bmaa.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) .and(bmaa.SERIES_ID.eq(seriesId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } .orderBy(lower(bmaa.NAME), bmaa.ROLE) @@ -108,7 +109,7 @@ class ReferentialDao( .from(bmaa) .apply { if (filterOnLibraryIds != null || filterBy?.type == FilterByType.LIBRARY) leftJoin(s).on(bmaa.SERIES_ID.eq(s.ID)) } .apply { if (filterBy?.type == FilterByType.COLLECTION) leftJoin(cs).on(bmaa.SERIES_ID.eq(cs.SERIES_ID)) } - .where(bmaa.NAME.containsIgnoreCase(search)) + .where(bmaa.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) .apply { role?.let { and(bmaa.ROLE.eq(role)) } } .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } .apply { @@ -142,7 +143,7 @@ class ReferentialDao( dsl.selectDistinct(a.NAME) .from(a) .apply { filterOnLibraryIds?.let { leftJoin(b).on(a.BOOK_ID.eq(b.ID)) } } - .where(a.NAME.containsIgnoreCase(search)) + .where(a.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) .apply { filterOnLibraryIds?.let { and(b.LIBRARY_ID.`in`(it)) } } .orderBy(a.NAME) .fetch(a.NAME) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt index 718d1781b..30a209348 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt @@ -2,6 +2,7 @@ package org.gotson.komga.infrastructure.jooq import org.gotson.komga.domain.model.SeriesCollection import org.gotson.komga.domain.persistence.SeriesCollectionRepository +import org.gotson.komga.infrastructure.language.stripAccents import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.CollectionRecord import org.jooq.DSLContext @@ -45,7 +46,7 @@ class SeriesCollectionDao( .firstOrNull() override fun searchAll(search: String?, pageable: Pageable): Page { - val conditions = search?.let { c.NAME.containsIgnoreCase(it) } + val conditions = search?.let { c.NAME.udfStripAccents().containsIgnoreCase(it.stripAccents()) } ?: DSL.trueCondition() val count = dsl.selectCount() @@ -76,7 +77,7 @@ class SeriesCollectionDao( .leftJoin(cs).on(c.ID.eq(cs.COLLECTION_ID)) .leftJoin(s).on(cs.SERIES_ID.eq(s.ID)) .where(s.LIBRARY_ID.`in`(belongsToLibraryIds)) - .apply { search?.let { and(c.NAME.containsIgnoreCase(it)) } } + .apply { search?.let { and(c.NAME.udfStripAccents().containsIgnoreCase(it.stripAccents())) } } .fetch(0, String::class.java) val count = ids.size @@ -86,7 +87,7 @@ class SeriesCollectionDao( val items = selectBase() .where(c.ID.`in`(ids)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .apply { search?.let { and(c.NAME.containsIgnoreCase(it)) } } + .apply { search?.let { and(c.NAME.udfStripAccents().containsIgnoreCase(it.stripAccents())) } } .orderBy(orderBy) .apply { if (pageable.isPaged) limit(pageable.pageSize).offset(pageable.offset) } .fetchAndMap(filterOnLibraryIds) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt index cf7d10984..93c8abd9d 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt @@ -3,6 +3,7 @@ package org.gotson.komga.infrastructure.jooq import org.gotson.komga.domain.model.ReadStatus import org.gotson.komga.domain.model.SeriesSearch import org.gotson.komga.domain.model.SeriesSearchWithReadProgress +import org.gotson.komga.infrastructure.language.stripAccents import org.gotson.komga.infrastructure.web.toFilePath import org.gotson.komga.interfaces.rest.dto.AuthorDto import org.gotson.komga.interfaces.rest.dto.BookMetadataAggregationDto @@ -229,7 +230,7 @@ class SeriesDtoDao( if (!libraryIds.isNullOrEmpty()) c = c.and(s.LIBRARY_ID.`in`(libraryIds)) if (!collectionIds.isNullOrEmpty()) c = c.and(cs.COLLECTION_ID.`in`(collectionIds)) - searchTerm?.let { c = c.and(d.TITLE.containsIgnoreCase(it)) } + searchTerm?.let { c = c.and(d.TITLE.udfStripAccents().containsIgnoreCase(it.stripAccents())) } searchRegex?.let { c = c.and((it.second.toColumn()).likeRegex(it.first)) } if (!metadataStatus.isNullOrEmpty()) c = c.and(d.STATUS.`in`(metadataStatus)) if (!publishers.isNullOrEmpty()) c = c.and(lower(d.PUBLISHER).`in`(publishers.map { it.lowercase() })) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt index ddd180989..0a2610f80 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt @@ -1,7 +1,10 @@ package org.gotson.komga.infrastructure.jooq +import org.gotson.komga.infrastructure.datasource.SqliteUdfDataSource import org.jooq.Field import org.jooq.SortField +import org.jooq.TableField +import org.jooq.impl.DSL import org.springframework.data.domain.Sort import java.time.LocalDateTime import java.time.ZoneId @@ -18,3 +21,6 @@ fun Sort.toOrderBy(sorts: Map>): List> fun LocalDateTime.toCurrentTimeZone(): LocalDateTime = this.atZone(ZoneId.of("Z")).withZoneSameInstant(ZoneId.systemDefault()).toLocalDateTime() + +fun TableField<*, String>.udfStripAccents() = + DSL.function(SqliteUdfDataSource.udfStripAccents, String::class.java, this) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/language/Utils.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/language/Utils.kt index de8443944..c47f0f899 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/language/Utils.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/language/Utils.kt @@ -1,5 +1,6 @@ package org.gotson.komga.infrastructure.language +import org.apache.commons.lang3.StringUtils import java.time.LocalDateTime import java.time.temporal.ChronoUnit import java.time.temporal.TemporalUnit @@ -36,3 +37,5 @@ fun Iterable.mostFrequent(transform: (T) -> R?): R? { fun LocalDateTime.notEquals(other: LocalDateTime, precision: TemporalUnit = ChronoUnit.MILLIS) = this.truncatedTo(precision) != other.truncatedTo(precision) + +fun String.stripAccents(): String = StringUtils.stripAccents(this)