From c2c697fba788f599f8743b1813c1c6c81db437b1 Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Mon, 30 Jun 2025 10:13:42 +0800 Subject: [PATCH] fix: don't strip accents on input data and sort series title with unicode collation --- .../komga/domain/service/SeriesLifecycle.kt | 2 +- .../infrastructure/jooq/main/BookDtoDao.kt | 2 +- .../infrastructure/jooq/main/SeriesDtoDao.kt | 3 +-- .../metadata/comicrack/ComicInfoProvider.kt | 3 +-- .../metadata/epub/EpubMetadataProvider.kt | 3 +-- .../metadata/mylar/MylarSeriesProvider.kt | 3 +-- .../gotson/komga/language/LanguageUtils.kt | 3 +++ .../domain/service/SeriesLifecycleTest.kt | 27 ------------------- .../jooq/main/SeriesDtoDaoTest.kt | 26 ++++++++++++++++++ .../comicrack/ComicInfoProviderTest.kt | 2 +- .../metadata/mylar/MylarSeriesProviderTest.kt | 2 +- 11 files changed, 37 insertions(+), 39 deletions(-) diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/SeriesLifecycle.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/SeriesLifecycle.kt index 7859ed30e..b014fba1a 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/SeriesLifecycle.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/SeriesLifecycle.kt @@ -155,7 +155,7 @@ class SeriesLifecycle( seriesMetadataRepository.insert( SeriesMetadata( title = series.name, - titleSort = series.name.stripAccents(), + titleSort = series.name, seriesId = series.id, ), ) diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/BookDtoDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/BookDtoDao.kt index 58b53d139..dc4163034 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/BookDtoDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/BookDtoDao.kt @@ -72,7 +72,7 @@ class BookDtoDao( private val sorts = mapOf( "name" to b.NAME.collate(SqliteUdfDataSource.COLLATION_UNICODE_3), - "series" to sd.TITLE_SORT.noCase(), + "series" to sd.TITLE_SORT.collate(SqliteUdfDataSource.COLLATION_UNICODE_3), "created" to b.CREATED_DATE, "createdDate" to b.CREATED_DATE, "lastModified" to b.LAST_MODIFIED_DATE, diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDao.kt index 1fa0a3b4c..f8651de05 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDao.kt @@ -10,7 +10,6 @@ import org.gotson.komga.infrastructure.jooq.SeriesSearchHelper import org.gotson.komga.infrastructure.jooq.csAlias import org.gotson.komga.infrastructure.jooq.inOrNoCondition import org.gotson.komga.infrastructure.jooq.insertTempStrings -import org.gotson.komga.infrastructure.jooq.noCase import org.gotson.komga.infrastructure.jooq.selectTempStrings import org.gotson.komga.infrastructure.jooq.sortByValues import org.gotson.komga.infrastructure.jooq.toSortField @@ -85,7 +84,7 @@ class SeriesDtoDao( private val sorts = mapOf( - "metadata.titleSort" to d.TITLE_SORT.noCase(), + "metadata.titleSort" to d.TITLE_SORT.collate(SqliteUdfDataSource.COLLATION_UNICODE_3), "createdDate" to s.CREATED_DATE, "created" to s.CREATED_DATE, "lastModifiedDate" to s.LAST_MODIFIED_DATE, diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/comicrack/ComicInfoProvider.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/comicrack/ComicInfoProvider.kt index 87235381e..b88eb6bf5 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/comicrack/ComicInfoProvider.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/comicrack/ComicInfoProvider.kt @@ -18,7 +18,6 @@ import org.gotson.komga.infrastructure.metadata.BookMetadataProvider import org.gotson.komga.infrastructure.metadata.SeriesMetadataFromBookProvider import org.gotson.komga.infrastructure.metadata.comicrack.dto.ComicInfo import org.gotson.komga.infrastructure.metadata.comicrack.dto.Manga -import org.gotson.komga.language.stripAccents import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.net.URI @@ -142,7 +141,7 @@ class ComicInfoProvider( return SeriesMetadataPatch( title = series, - titleSort = series?.stripAccents(), + titleSort = series, status = null, summary = null, readingDirection = readingDirection, diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/epub/EpubMetadataProvider.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/epub/EpubMetadataProvider.kt index 619b142b2..4249f017f 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/epub/EpubMetadataProvider.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/epub/EpubMetadataProvider.kt @@ -14,7 +14,6 @@ import org.gotson.komga.domain.model.SeriesMetadataPatch import org.gotson.komga.infrastructure.mediacontainer.epub.getPackageFileContent import org.gotson.komga.infrastructure.metadata.BookMetadataProvider import org.gotson.komga.infrastructure.metadata.SeriesMetadataFromBookProvider -import org.gotson.komga.language.stripAccents import org.jsoup.Jsoup import org.jsoup.parser.Parser import org.jsoup.safety.Safelist @@ -138,7 +137,7 @@ class EpubMetadataProvider( return SeriesMetadataPatch( title = series, - titleSort = series?.stripAccents(), + titleSort = series, status = null, readingDirection = direction, publisher = publisher, diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/mylar/MylarSeriesProvider.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/mylar/MylarSeriesProvider.kt index 7d3587118..f67742f4c 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/mylar/MylarSeriesProvider.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/mylar/MylarSeriesProvider.kt @@ -11,7 +11,6 @@ import org.gotson.komga.domain.model.Sidecar import org.gotson.komga.infrastructure.metadata.SeriesMetadataProvider import org.gotson.komga.infrastructure.metadata.mylar.dto.Status import org.gotson.komga.infrastructure.sidecar.SidecarSeriesConsumer -import org.gotson.komga.language.stripAccents import org.springframework.stereotype.Service import kotlin.io.path.notExists import org.gotson.komga.infrastructure.metadata.mylar.dto.Series as MylarSeries @@ -47,7 +46,7 @@ class MylarSeriesProvider( return SeriesMetadataPatch( title = title, - titleSort = title.stripAccents(), + titleSort = title, status = when (metadata.status) { Status.Ended -> SeriesMetadata.Status.ENDED diff --git a/komga/src/main/kotlin/org/gotson/komga/language/LanguageUtils.kt b/komga/src/main/kotlin/org/gotson/komga/language/LanguageUtils.kt index 4ab22e9e0..ed63190b8 100644 --- a/komga/src/main/kotlin/org/gotson/komga/language/LanguageUtils.kt +++ b/komga/src/main/kotlin/org/gotson/komga/language/LanguageUtils.kt @@ -46,6 +46,9 @@ fun LocalDateTime.notEquals( precision: TemporalUnit = ChronoUnit.MILLIS, ) = this.truncatedTo(precision) != other.truncatedTo(precision) +/** + * Warning: This affects the Unicode code points of Korean Hangul. + */ fun String.stripAccents(): String = StringUtils.stripAccents(this) fun LocalDate.toDate(): Date = Date.from(this.atStartOfDay(ZoneId.of("Z")).toInstant()) diff --git a/komga/src/test/kotlin/org/gotson/komga/domain/service/SeriesLifecycleTest.kt b/komga/src/test/kotlin/org/gotson/komga/domain/service/SeriesLifecycleTest.kt index 3ab79aae9..f87d993a5 100644 --- a/komga/src/test/kotlin/org/gotson/komga/domain/service/SeriesLifecycleTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/domain/service/SeriesLifecycleTest.kt @@ -168,33 +168,6 @@ class SeriesLifecycleTest( assertThat(savedBooks.map { it.number }).containsExactly(1, 2, 3, 4, 5) } - @Test - fun `given series name with diacritics when creating series then diacritics are stripped from metadata titlesort`() { - // given - val series1 = makeSeries("À l'assaut", library.id) - val series2 = makeSeries("Être ou ne pas être", library.id) - val series3 = makeSeries("Écarlate", library.id) - - // when - val created1 = seriesLifecycle.createSeries(series1) - val created2 = seriesLifecycle.createSeries(series2) - val created3 = seriesLifecycle.createSeries(series3) - - // then - with(seriesMetadataRepository.findById(created1.id)) { - assertThat(title).isEqualTo(series1.name) - assertThat(titleSort).isEqualTo("A l'assaut") - } - with(seriesMetadataRepository.findById(created2.id)) { - assertThat(title).isEqualTo(series2.name) - assertThat(titleSort).isEqualTo("Etre ou ne pas etre") - } - with(seriesMetadataRepository.findById(created3.id)) { - assertThat(title).isEqualTo(series3.name) - assertThat(titleSort).isEqualTo("Ecarlate") - } - } - @Nested inner class Transactions { @Test diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDaoTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDaoTest.kt index 15d9bf35b..f126e23fb 100644 --- a/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDaoTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/main/SeriesDtoDaoTest.kt @@ -127,6 +127,32 @@ class SeriesDtoDaoTest( } } + @Nested + inner class SortCriteria { + @Test + fun `given series when sorting by title sort then results are ordered`() { + // given + seriesLifecycle.createSeries(makeSeries("Éb", library.id)) + seriesLifecycle.createSeries(makeSeries("Ea", library.id)) + seriesLifecycle.createSeries(makeSeries("Ec", library.id)) + + searchIndexLifecycle.rebuildIndex() + Thread.sleep(500) // index rebuild is done asynchronously, and need a slight delay to be updated + + // when + val found = + seriesDtoDao + .findAll( + SeriesSearch(), + SearchContext(user), + UnpagedSorted(Sort.by("metadata.titleSort")), + ).content + + // then + assertThat(found.map { it.metadata.title }).containsExactly("Ea", "Éb", "Ec") + } + } + @Nested inner class ReadProgress { @Test diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/metadata/comicrack/ComicInfoProviderTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/metadata/comicrack/ComicInfoProviderTest.kt index fdee1766b..129c8e9ed 100644 --- a/komga/src/test/kotlin/org/gotson/komga/infrastructure/metadata/comicrack/ComicInfoProviderTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/metadata/comicrack/ComicInfoProviderTest.kt @@ -389,7 +389,7 @@ class ComicInfoProviderTest { with(patch) { assertThat(title).isEqualTo("séries") - assertThat(titleSort).isEqualTo("series") + assertThat(titleSort).isEqualTo("séries") assertThat(status).isNull() assertThat(collections).containsExactlyInAnyOrder("collections", "multiple") assertThat(publisher).isEqualTo("publisher") diff --git a/komga/src/test/kotlin/org/gotson/komga/infrastructure/metadata/mylar/MylarSeriesProviderTest.kt b/komga/src/test/kotlin/org/gotson/komga/infrastructure/metadata/mylar/MylarSeriesProviderTest.kt index 8a3582204..38724f95d 100644 --- a/komga/src/test/kotlin/org/gotson/komga/infrastructure/metadata/mylar/MylarSeriesProviderTest.kt +++ b/komga/src/test/kotlin/org/gotson/komga/infrastructure/metadata/mylar/MylarSeriesProviderTest.kt @@ -60,7 +60,7 @@ class MylarSeriesProviderTest { with(patch) { assertThat(title).isEqualTo("Sàndman") - assertThat(titleSort).isEqualTo("Sandman") + assertThat(titleSort).isEqualTo("Sàndman") assertThat(status).isEqualTo(SeriesMetadata.Status.ENDED) assertThat(summary).isEqualTo("Sandman comics formatted") assertThat(readingDirection).isNull()