diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/datasource/SqliteUdfDataSource.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/datasource/SqliteUdfDataSource.kt index e69fc7955..d4475040e 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/datasource/SqliteUdfDataSource.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/datasource/SqliteUdfDataSource.kt @@ -2,7 +2,7 @@ package org.gotson.komga.infrastructure.datasource import com.ibm.icu.text.Collator import io.github.oshai.kotlinlogging.KotlinLogging -import org.gotson.komga.language.stripAccents +import org.gotson.komga.infrastructure.unicode.Collators import org.sqlite.Collation import org.sqlite.Function import org.sqlite.SQLiteConnection @@ -13,7 +13,7 @@ private val log = KotlinLogging.logger {} class SqliteUdfDataSource : SQLiteDataSource() { companion object { - const val UDF_STRIP_ACCENTS = "UDF_STRIP_ACCENTS" + const val COLLATION_UNICODE_1 = "COLLATION_UNICODE_1" const val COLLATION_UNICODE_3 = "COLLATION_UNICODE_3" } @@ -26,8 +26,8 @@ class SqliteUdfDataSource : SQLiteDataSource() { private fun addAllUdf(connection: SQLiteConnection) { createUdfRegexp(connection) - createUdfStripAccents(connection) - createUnicode3Collation(connection) + createUnicodeCollation(connection, COLLATION_UNICODE_3, Collators.collator3) + createUnicodeCollation(connection, COLLATION_UNICODE_1, Collators.collator1) } private fun createUdfRegexp(connection: SQLiteConnection) { @@ -46,33 +46,16 @@ class SqliteUdfDataSource : SQLiteDataSource() { ) } - private fun createUdfStripAccents(connection: SQLiteConnection) { - log.debug { "Adding custom $UDF_STRIP_ACCENTS function" } - Function.create( - connection, - UDF_STRIP_ACCENTS, - object : Function() { - override fun xFunc() = - when (val text = value_text(0)) { - null -> error("Argument must not be null") - else -> result(text.stripAccents()) - } - }, - ) - } - - private fun createUnicode3Collation(connection: SQLiteConnection) { - log.debug { "Adding custom $COLLATION_UNICODE_3 collation" } + private fun createUnicodeCollation( + connection: SQLiteConnection, + collationName: String, + collator: Collator, + ) { + log.debug { "Adding custom $collationName collation" } Collation.create( connection, - COLLATION_UNICODE_3, + collationName, object : Collation() { - val collator = - Collator.getInstance().apply { - strength = Collator.TERTIARY - decomposition = Collator.CANONICAL_DECOMPOSITION - } - override fun xCompare( str1: String, str2: String, diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookSearchHelper.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookSearchHelper.kt index 232c547ac..12ba9663e 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookSearchHelper.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookSearchHelper.kt @@ -7,7 +7,6 @@ import org.gotson.komga.domain.model.ReadStatus import org.gotson.komga.domain.model.SearchCondition import org.gotson.komga.domain.model.SearchContext import org.gotson.komga.domain.model.SearchOperator -import org.gotson.komga.infrastructure.datasource.SqliteUdfDataSource import org.gotson.komga.infrastructure.jooq.RequiredJoin.ReadProgress import org.gotson.komga.jooq.main.Tables import org.jooq.Condition @@ -147,11 +146,7 @@ class BookSearchHelper( DSL .select(Tables.BOOK_METADATA_TAG.BOOK_ID) .from(Tables.BOOK_METADATA_TAG) - .where( - Tables.BOOK_METADATA_TAG.TAG - .collate(SqliteUdfDataSource.COLLATION_UNICODE_3) - .equalIgnoreCase(tag), - ) + .where(Tables.BOOK_METADATA_TAG.TAG.unicode1().equal(tag)) } val innerAny = { DSL @@ -175,21 +170,8 @@ class BookSearchHelper( .select(Tables.BOOK_METADATA_AUTHOR.BOOK_ID) .from(Tables.BOOK_METADATA_AUTHOR) .where(DSL.noCondition()) - .apply { - if (name != null) - and( - Tables.BOOK_METADATA_AUTHOR.NAME - .collate(SqliteUdfDataSource.COLLATION_UNICODE_3) - .equalIgnoreCase(name), - ) - }.apply { - if (role != null) - and( - Tables.BOOK_METADATA_AUTHOR.ROLE - .collate(SqliteUdfDataSource.COLLATION_UNICODE_3) - .equalIgnoreCase(role), - ) - } + .apply { if (name != null) and(Tables.BOOK_METADATA_AUTHOR.NAME.unicode1().equal(name)) } + .apply { if (role != null) and(Tables.BOOK_METADATA_AUTHOR.ROLE.unicode1().equal(role)) } } when (searchCondition.operator) { is SearchOperator.Is -> { diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesSearchHelper.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesSearchHelper.kt index 9de826df3..07f79decd 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesSearchHelper.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesSearchHelper.kt @@ -101,19 +101,12 @@ class SeriesSearchHelper( DSL .select(Tables.SERIES_METADATA_TAG.SERIES_ID) .from(Tables.SERIES_METADATA_TAG) - .where( - Tables.SERIES_METADATA_TAG.TAG - .collate(SqliteUdfDataSource.COLLATION_UNICODE_3) - .equalIgnoreCase(tag), - ).union( + .where(Tables.SERIES_METADATA_TAG.TAG.unicode1().equal(tag)) + .union( DSL .select(Tables.BOOK_METADATA_AGGREGATION_TAG.SERIES_ID) .from(Tables.BOOK_METADATA_AGGREGATION_TAG) - .where( - Tables.BOOK_METADATA_AGGREGATION_TAG.TAG - .collate(SqliteUdfDataSource.COLLATION_UNICODE_3) - .equalIgnoreCase(tag), - ), + .where(Tables.BOOK_METADATA_AGGREGATION_TAG.TAG.unicode1().equal(tag)), ) } val innerAny = { @@ -216,11 +209,7 @@ class SeriesSearchHelper( DSL .select(Tables.SERIES_METADATA_GENRE.SERIES_ID) .from(Tables.SERIES_METADATA_GENRE) - .where( - Tables.SERIES_METADATA_GENRE.GENRE - .collate(SqliteUdfDataSource.COLLATION_UNICODE_3) - .equalIgnoreCase(genre), - ) + .where(Tables.SERIES_METADATA_GENRE.GENRE.unicode1().equal(genre)) } val innerAny = { DSL @@ -247,11 +236,7 @@ class SeriesSearchHelper( DSL .select(Tables.SERIES_METADATA_SHARING.SERIES_ID) .from(Tables.SERIES_METADATA_SHARING) - .where( - Tables.SERIES_METADATA_SHARING.LABEL - .collate(SqliteUdfDataSource.COLLATION_UNICODE_3) - .equalIgnoreCase(label), - ) + .where(Tables.SERIES_METADATA_SHARING.LABEL.unicode1().equal(label)) } val innerAny = { DSL diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt index bde233857..b7bd7f9d2 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt @@ -17,6 +17,10 @@ import java.util.zip.GZIPOutputStream fun Field.noCase() = this.collate("NOCASE") +fun Field.unicode1() = this.collate(SqliteUdfDataSource.COLLATION_UNICODE_1) + +fun Field.unicode3() = this.collate(SqliteUdfDataSource.COLLATION_UNICODE_3) + fun Sort.toOrderBy(sorts: Map>): List> = this.mapNotNull { it.toSortField(sorts) @@ -44,8 +48,6 @@ fun Field.inOrNoCondition(list: Collection?): Condition = else -> this.`in`(list) } -fun Field.udfStripAccents() = DSL.function(SqliteUdfDataSource.UDF_STRIP_ACCENTS, String::class.java, this) - fun ContentRestrictions.toCondition(): Condition { val ageAllowed = if (ageRestriction?.restriction == AllowExclude.ALLOW_ONLY) { diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/BookDtoDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/BookDtoDao.kt index 7231899da..ad89d3a5d 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/BookDtoDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/BookDtoDao.kt @@ -4,7 +4,6 @@ import org.gotson.komga.domain.model.BookSearch import org.gotson.komga.domain.model.ContentRestrictions import org.gotson.komga.domain.model.ReadList import org.gotson.komga.domain.model.SearchContext -import org.gotson.komga.infrastructure.datasource.SqliteUdfDataSource import org.gotson.komga.infrastructure.jooq.BookSearchHelper import org.gotson.komga.infrastructure.jooq.RequiredJoin import org.gotson.komga.infrastructure.jooq.SplitDslDaoBase @@ -16,6 +15,7 @@ import org.gotson.komga.infrastructure.jooq.sortByValues import org.gotson.komga.infrastructure.jooq.toCondition import org.gotson.komga.infrastructure.jooq.toOrderBy import org.gotson.komga.infrastructure.jooq.toSortField +import org.gotson.komga.infrastructure.jooq.unicode3 import org.gotson.komga.infrastructure.search.LuceneEntity import org.gotson.komga.infrastructure.search.LuceneHelper import org.gotson.komga.infrastructure.web.toFilePath @@ -73,8 +73,8 @@ class BookDtoDao( private val sorts = mapOf( - "name" to b.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3), - "series" to sd.TITLE_SORT.collate(SqliteUdfDataSource.COLLATION_UNICODE_3), + "name" to b.NAME.unicode3(), + "series" to sd.TITLE_SORT.unicode3(), "created" to b.CREATED_DATE, "createdDate" to b.CREATED_DATE, "lastModified" to b.LAST_MODIFIED_DATE, @@ -87,7 +87,7 @@ class BookDtoDao( "media.comment" to m.COMMENT.noCase(), "media.mediaType" to m.MEDIA_TYPE.noCase(), "media.pagesCount" to m.PAGE_COUNT, - "metadata.title" to d.TITLE.collate(SqliteUdfDataSource.COLLATION_UNICODE_3), + "metadata.title" to d.TITLE.unicode3(), "metadata.numberSort" to d.NUMBER_SORT, "metadata.releaseDate" to d.RELEASE_DATE, "readProgress.lastModified" to r.LAST_MODIFIED_DATE, diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/ReadListDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/ReadListDao.kt index b869509b8..29eaf090d 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/ReadListDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/ReadListDao.kt @@ -10,6 +10,7 @@ import org.gotson.komga.infrastructure.jooq.inOrNoCondition import org.gotson.komga.infrastructure.jooq.sortByValues import org.gotson.komga.infrastructure.jooq.toCondition import org.gotson.komga.infrastructure.jooq.toSortField +import org.gotson.komga.infrastructure.jooq.unicode3 import org.gotson.komga.infrastructure.search.LuceneEntity import org.gotson.komga.infrastructure.search.LuceneHelper import org.gotson.komga.jooq.main.Tables @@ -46,7 +47,7 @@ class ReadListDao( private val sorts = mapOf( - "name" to rl.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3), + "name" to rl.NAME.unicode3(), "createdDate" to rl.CREATED_DATE, "lastModifiedDate" to rl.LAST_MODIFIED_DATE, ) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/ReferentialDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/ReferentialDao.kt index 35ad984b7..dd61173ca 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/ReferentialDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/ReferentialDao.kt @@ -2,9 +2,9 @@ package org.gotson.komga.infrastructure.jooq.main import org.gotson.komga.domain.model.Author import org.gotson.komga.domain.persistence.ReferentialRepository -import org.gotson.komga.infrastructure.datasource.SqliteUdfDataSource import org.gotson.komga.infrastructure.jooq.SplitDslDaoBase -import org.gotson.komga.infrastructure.jooq.udfStripAccents +import org.gotson.komga.infrastructure.jooq.unicode1 +import org.gotson.komga.infrastructure.jooq.unicode3 import org.gotson.komga.jooq.main.Tables import org.gotson.komga.jooq.main.tables.records.BookMetadataAggregationAuthorRecord import org.gotson.komga.jooq.main.tables.records.BookMetadataAuthorRecord @@ -49,9 +49,9 @@ class ReferentialDao( .selectDistinct(a.NAME, a.ROLE) .from(a) .apply { filterOnLibraryIds?.let { leftJoin(b).on(a.BOOK_ID.eq(b.ID)) } } - .where(a.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) + .where(a.NAME.unicode1().contains(search)) .apply { filterOnLibraryIds?.let { and(b.LIBRARY_ID.`in`(it)) } } - .orderBy(a.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(a.NAME.unicode3()) .fetchInto(a) .map { it.toDomain() } @@ -65,10 +65,10 @@ class ReferentialDao( .from(bmaa) .leftJoin(s) .on(bmaa.SERIES_ID.eq(s.ID)) - .where(bmaa.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) + .where(bmaa.NAME.unicode1().contains(search)) .and(s.LIBRARY_ID.eq(libraryId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(bmaa.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(bmaa.NAME.unicode3()) .fetchInto(bmaa) .map { it.toDomain() } @@ -83,10 +83,10 @@ class ReferentialDao( .leftJoin(cs) .on(bmaa.SERIES_ID.eq(cs.SERIES_ID)) .apply { filterOnLibraryIds?.let { leftJoin(s).on(bmaa.SERIES_ID.eq(s.ID)) } } - .where(bmaa.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) + .where(bmaa.NAME.unicode1().contains(search)) .and(cs.COLLECTION_ID.eq(collectionId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(bmaa.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(bmaa.NAME.unicode3()) .fetchInto(bmaa) .map { it.toDomain() } @@ -99,10 +99,10 @@ class ReferentialDao( .selectDistinct(bmaa.NAME, bmaa.ROLE) .from(bmaa) .apply { filterOnLibraryIds?.let { leftJoin(s).on(bmaa.SERIES_ID.eq(s.ID)) } } - .where(bmaa.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) + .where(bmaa.NAME.unicode1().contains(search)) .and(bmaa.SERIES_ID.eq(seriesId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(bmaa.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(bmaa.NAME.unicode3()) .fetchInto(bmaa) .map { it.toDomain() } @@ -177,7 +177,7 @@ class ReferentialDao( .leftJoin(rb) .on(b.ID.eq(rb.BOOK_ID)) }.where(noCondition()) - .apply { search?.let { and(bmaa.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) } } + .apply { search?.let { and(bmaa.NAME.unicode1().contains(search)) } } .apply { role?.let { and(bmaa.ROLE.eq(role)) } } .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } .apply { @@ -192,7 +192,7 @@ class ReferentialDao( } val count = dslRO.fetchCount(query) - val sort = bmaa.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3) + val sort = bmaa.NAME.unicode3() val items = query @@ -220,9 +220,9 @@ class ReferentialDao( .selectDistinct(a.NAME) .from(a) .apply { filterOnLibraryIds?.let { leftJoin(b).on(a.BOOK_ID.eq(b.ID)) } } - .where(a.NAME.udfStripAccents().containsIgnoreCase(search.stripAccents())) + .where(a.NAME.unicode1().contains(search)) .apply { filterOnLibraryIds?.let { and(b.LIBRARY_ID.`in`(it)) } } - .orderBy(a.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(a.NAME.unicode3()) .fetch(a.NAME) override fun findAllAuthorsRoles(filterOnLibraryIds: Collection?): List = @@ -248,7 +248,7 @@ class ReferentialDao( .on(g.SERIES_ID.eq(s.ID)) .where(s.LIBRARY_ID.`in`(it)) } - }.orderBy(g.GENRE.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + }.orderBy(g.GENRE.unicode3()) .fetchSet(g.GENRE) override fun findAllGenresByLibraries( @@ -262,7 +262,7 @@ class ReferentialDao( .on(g.SERIES_ID.eq(s.ID)) .where(s.LIBRARY_ID.`in`(libraryIds)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(g.GENRE.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(g.GENRE.unicode3()) .fetchSet(g.GENRE) override fun findAllGenresByCollection( @@ -277,7 +277,7 @@ class ReferentialDao( .apply { filterOnLibraryIds?.let { leftJoin(s).on(g.SERIES_ID.eq(s.ID)) } } .where(cs.COLLECTION_ID.eq(collectionId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(g.GENRE.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(g.GENRE.unicode3()) .fetchSet(g.GENRE) override fun findAllSeriesAndBookTags(filterOnLibraryIds: Collection?): Set = @@ -351,7 +351,7 @@ class ReferentialDao( .on(st.SERIES_ID.eq(s.ID)) .where(s.LIBRARY_ID.`in`(it)) } - }.orderBy(st.TAG.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + }.orderBy(st.TAG.unicode3()) .fetchSet(st.TAG) override fun findAllSeriesTagsByLibrary( @@ -365,7 +365,7 @@ class ReferentialDao( .on(st.SERIES_ID.eq(s.ID)) .where(s.LIBRARY_ID.eq(libraryId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(st.TAG.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(st.TAG.unicode3()) .fetchSet(st.TAG) override fun findAllBookTagsBySeries( @@ -379,7 +379,7 @@ class ReferentialDao( .on(bt.BOOK_ID.eq(b.ID)) .where(b.SERIES_ID.eq(seriesId)) .apply { filterOnLibraryIds?.let { and(b.LIBRARY_ID.`in`(it)) } } - .orderBy(bt.TAG.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(bt.TAG.unicode3()) .fetchSet(bt.TAG) override fun findAllBookTagsByReadList( @@ -395,7 +395,7 @@ class ReferentialDao( .on(bt.BOOK_ID.eq(rb.BOOK_ID)) .where(rb.READLIST_ID.eq(readListId)) .apply { filterOnLibraryIds?.let { and(b.LIBRARY_ID.`in`(it)) } } - .orderBy(bt.TAG.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(bt.TAG.unicode3()) .fetchSet(bt.TAG) override fun findAllSeriesTagsByCollection( @@ -410,7 +410,7 @@ class ReferentialDao( .apply { filterOnLibraryIds?.let { leftJoin(s).on(st.SERIES_ID.eq(s.ID)) } } .where(cs.COLLECTION_ID.eq(collectionId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(st.TAG.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(st.TAG.unicode3()) .fetchSet(st.TAG) override fun findAllBookTags(filterOnLibraryIds: Collection?): Set = @@ -423,7 +423,7 @@ class ReferentialDao( .on(bt.BOOK_ID.eq(b.ID)) .where(b.LIBRARY_ID.`in`(it)) } - }.orderBy(bt.TAG.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + }.orderBy(bt.TAG.unicode3()) .fetchSet(bt.TAG) override fun findAllLanguages(filterOnLibraryIds: Collection?): Set = @@ -474,7 +474,7 @@ class ReferentialDao( .apply { filterOnLibraryIds?.let { leftJoin(s).on(sd.SERIES_ID.eq(s.ID)) } } .where(sd.PUBLISHER.ne("")) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(sd.PUBLISHER.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(sd.PUBLISHER.unicode3()) .fetchSet(sd.PUBLISHER) override fun findAllPublishers( @@ -490,7 +490,7 @@ class ReferentialDao( .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } val count = dslRO.fetchCount(query) - val sort = sd.PUBLISHER.collate(SqliteUdfDataSource.COLLATION_UNICODE_3) + val sort = sd.PUBLISHER.unicode3() val items = query @@ -521,7 +521,7 @@ class ReferentialDao( .where(sd.PUBLISHER.ne("")) .and(s.LIBRARY_ID.`in`(libraryIds)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(sd.PUBLISHER.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(sd.PUBLISHER.unicode3()) .fetchSet(sd.PUBLISHER) override fun findAllPublishersByCollection( @@ -537,7 +537,7 @@ class ReferentialDao( .where(sd.PUBLISHER.ne("")) .and(cs.COLLECTION_ID.eq(collectionId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(sd.PUBLISHER.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(sd.PUBLISHER.unicode3()) .fetchSet(sd.PUBLISHER) override fun findAllAgeRatings(filterOnLibraryIds: Collection?): Set = @@ -633,7 +633,7 @@ class ReferentialDao( .on(sl.SERIES_ID.eq(s.ID)) .where(s.LIBRARY_ID.`in`(it)) } - }.orderBy(sl.LABEL.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + }.orderBy(sl.LABEL.unicode3()) .fetchSet(sl.LABEL) override fun findAllSharingLabelsByLibraries( @@ -647,7 +647,7 @@ class ReferentialDao( .on(sl.SERIES_ID.eq(s.ID)) .where(s.LIBRARY_ID.`in`(libraryIds)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(sl.LABEL.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(sl.LABEL.unicode3()) .fetchSet(sl.LABEL) override fun findAllSharingLabelsByCollection( @@ -662,7 +662,7 @@ class ReferentialDao( .apply { filterOnLibraryIds?.let { leftJoin(s).on(sl.SERIES_ID.eq(s.ID)) } } .where(cs.COLLECTION_ID.eq(collectionId)) .apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } } - .orderBy(sl.LABEL.collate(SqliteUdfDataSource.COLLATION_UNICODE_3)) + .orderBy(sl.LABEL.unicode3()) .fetchSet(sl.LABEL) private fun BookMetadataAuthorRecord.toDomain(): Author = diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesCollectionDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesCollectionDao.kt index c89a125ba..fe47c3982 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesCollectionDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesCollectionDao.kt @@ -10,6 +10,7 @@ import org.gotson.komga.infrastructure.jooq.inOrNoCondition import org.gotson.komga.infrastructure.jooq.sortByValues import org.gotson.komga.infrastructure.jooq.toCondition import org.gotson.komga.infrastructure.jooq.toSortField +import org.gotson.komga.infrastructure.jooq.unicode3 import org.gotson.komga.infrastructure.search.LuceneEntity import org.gotson.komga.infrastructure.search.LuceneHelper import org.gotson.komga.jooq.main.Tables @@ -45,7 +46,7 @@ class SeriesCollectionDao( private val sorts = mapOf( - "name" to c.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3), + "name" to c.NAME.unicode3(), ) override fun findByIdOrNull( diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDao.kt index 7157afaad..c5df5ef4d 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDao.kt @@ -12,6 +12,7 @@ import org.gotson.komga.infrastructure.jooq.csAlias import org.gotson.komga.infrastructure.jooq.inOrNoCondition import org.gotson.komga.infrastructure.jooq.sortByValues import org.gotson.komga.infrastructure.jooq.toSortField +import org.gotson.komga.infrastructure.jooq.unicode3 import org.gotson.komga.infrastructure.search.LuceneEntity import org.gotson.komga.infrastructure.search.LuceneHelper import org.gotson.komga.infrastructure.web.toFilePath @@ -82,7 +83,7 @@ class SeriesDtoDao( private val sorts = mapOf( - "metadata.titleSort" to d.TITLE_SORT.collate(SqliteUdfDataSource.COLLATION_UNICODE_3), + "metadata.titleSort" to d.TITLE_SORT.unicode3(), "createdDate" to s.CREATED_DATE, "created" to s.CREATED_DATE, "lastModifiedDate" to s.LAST_MODIFIED_DATE, @@ -90,7 +91,7 @@ class SeriesDtoDao( "booksMetadata.releaseDate" to bma.RELEASE_DATE, "readDate" to rs.MOST_RECENT_READ_DATE, "collection.number" to cs.NUMBER, - "name" to s.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3), + "name" to s.NAME.unicode3(), "booksCount" to s.BOOK_COUNT, "random" to DSL.rand(), ) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/unicode/Collators.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/unicode/Collators.kt new file mode 100644 index 000000000..b5934e7f5 --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/unicode/Collators.kt @@ -0,0 +1,23 @@ +package org.gotson.komga.infrastructure.unicode + +import com.ibm.icu.text.Collator + +object Collators { + /** + * Used for matching + */ + val collator1: Collator = + Collator.getInstance().apply { + strength = Collator.PRIMARY + decomposition = Collator.CANONICAL_DECOMPOSITION + } + + /** + * Used for sorting + */ + val collator3: Collator = + Collator.getInstance().apply { + strength = Collator.TERTIARY + decomposition = Collator.CANONICAL_DECOMPOSITION + } +} diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/BookSearchTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/BookSearchTest.kt index 8bc01bb4d..3b969f041 100644 --- a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/BookSearchTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/BookSearchTest.kt @@ -534,7 +534,7 @@ class BookSearchTest( } run { - val search = BookSearch(SearchCondition.Tag(SearchOperator.Is("FICTION"))) + val search = BookSearch(SearchCondition.Tag(SearchOperator.Is("FICTîON"))) val found = bookDao.findAll(search.condition, SearchContext(user1), Pageable.unpaged()).content val foundDto = bookDtoDao.findAll(search, SearchContext(user1), Pageable.unpaged()).content @@ -870,7 +870,7 @@ class BookSearchTest( run { val search = BookSearch( - SearchCondition.Author(SearchOperator.Is(AuthorMatch("john"))), + SearchCondition.Author(SearchOperator.Is(AuthorMatch("jÒhn"))), ) val found = bookDao.findAll(search.condition, SearchContext(user1), Pageable.unpaged()).content val foundDto = bookDtoDao.findAll(search, SearchContext(user1), Pageable.unpaged()).content @@ -883,7 +883,7 @@ class BookSearchTest( run { val search = BookSearch( - SearchCondition.Author(SearchOperator.Is(AuthorMatch("john", "writer"))), + SearchCondition.Author(SearchOperator.Is(AuthorMatch("john", "WRÎTER"))), ) val found = bookDao.findAll(search.condition, SearchContext(user1), Pageable.unpaged()).content val foundDto = bookDtoDao.findAll(search, SearchContext(user1), Pageable.unpaged()).content diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesSearchTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesSearchTest.kt index 72a372f6d..f69bdd82e 100644 --- a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesSearchTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesSearchTest.kt @@ -439,7 +439,7 @@ class SeriesSearchTest( } run { - val search = SeriesSearch(SearchCondition.Tag(SearchOperator.Is("FICTION"))) + val search = SeriesSearch(SearchCondition.Tag(SearchOperator.Is("FîCTION"))) val found = seriesDao.findAll(search.condition, SearchContext(user1), Pageable.unpaged()).content val foundDto = seriesDtoDao.findAll(search, SearchContext(user1), Pageable.unpaged()).content @@ -453,7 +453,7 @@ class SeriesSearchTest( SearchCondition.AllOfSeries( listOf( SearchCondition.Tag(SearchOperator.Is("FICTION")), - SearchCondition.Tag(SearchOperator.Is("horror")), + SearchCondition.Tag(SearchOperator.Is("hórror")), ), ), ) @@ -535,7 +535,7 @@ class SeriesSearchTest( } makeSeries("2", library2.id).let { series -> seriesLifecycle.createSeries(series) - seriesMetadataRepository.findById(series.id).let { seriesMetadataRepository.update(it.copy(sharingLabels = setOf("kids"))) } + seriesMetadataRepository.findById(series.id).let { seriesMetadataRepository.update(it.copy(sharingLabels = setOf("kÎds"))) } } makeSeries("3", library1.id).let { series -> seriesLifecycle.createSeries(series) @@ -708,7 +708,7 @@ class SeriesSearchTest( } makeSeries("2", library2.id).let { series -> seriesLifecycle.createSeries(series) - seriesMetadataRepository.findById(series.id).let { seriesMetadataRepository.update(it.copy(genres = setOf("kids"))) } + seriesMetadataRepository.findById(series.id).let { seriesMetadataRepository.update(it.copy(genres = setOf("kÎds"))) } } makeSeries("3", library1.id).let { series -> seriesLifecycle.createSeries(series) diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/unicode/CollatorsTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/unicode/CollatorsTest.kt new file mode 100644 index 000000000..5bffd8656 --- /dev/null +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/unicode/CollatorsTest.kt @@ -0,0 +1,23 @@ +package org.gotson.komga.infrastructure.unicode + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class CollatorsTest { + @Test + fun collator1() { + assertThat(Collators.collator1.compare("café", "cafe")).isEqualTo(0) // accents ignored + assertThat(Collators.collator1.compare("CAFE", "cafe")).isEqualTo(0) // case ignored + assertThat(Collators.collator1.compare("안녕하세요", "안녕하세요")).isEqualTo(0) // hangul + assertThat(Collators.collator1.compare("あ", "ア")).isEqualTo(0) // katakana = hiragana + assertThat(Collators.collator1.compare("が", "か")).isEqualTo(0) // dakuten + } + + @Test + fun collator3() { + assertThat(Collators.collator3.compare("café", "cafe")).isNotEqualTo(0) // accents not ignored + assertThat(Collators.collator3.compare("CAFE", "cafe")).isNotEqualTo(0) // case not ignored + assertThat(Collators.collator3.compare("안녕하세요", "안녕하세요")).isEqualTo(0) // hangul + assertThat(Collators.collator3.compare("あ", "ア")).isNotEqualTo(0) // katakana != hiragana + } +}