Backup_Repos/komga
1
0
Fork 0
mirror of https://github.com/gotson/komga.git synced 2025-12-16 13:33:49 +01:00
Code Issues Projects Releases Wiki Activity

feat: aggregate book tags at series level

Browse source 
closes #513
This commit is contained in:
Gauthier Roebroeck 2021-07-30 14:58:48 +08:00
parent 64acfeff99
commit 0c9a063cc3
12 changed files with 138 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
komga/src/flyway/resources/db/migration/sqlite/V20210730111835__book_metadata_aggregation_tag.sql Normal file
View file

@ -0,0 +1,12 @@
CREATE TABLE BOOK_METADATA_AGGREGATION_TAG
(
TAG varchar NOT NULL,
SERIES_ID varchar NOT NULL,
FOREIGN KEY (SERIES_ID) REFERENCES SERIES (ID)
);
-- aggregate existing data
insert into BOOK_METADATA_AGGREGATION_TAG
select distinct bt.TAG, b.SERIES_ID
from BOOK_METADATA_TAG bt
left join BOOK B on B.ID = bt.BOOK_ID;

1
komga/src/main/kotlin/org/gotson/komga/domain/model/BookMetadataAggregation.kt
View file

@ -5,6 +5,7 @@ import java.time.LocalDateTime
data class BookMetadataAggregation(
val authors: List<Author> = emptyList(),
val tags: Set<String> = emptySet(),
val releaseDate: LocalDate? = null,
val summary: String = "",
val summaryNumber: String = "",

2
komga/src/main/kotlin/org/gotson/komga/domain/persistence/ReferentialRepository.kt
View file

@ -23,6 +23,8 @@ interface ReferentialRepository {
fun findAllGenresByCollection(collectionId: String, filterOnLibraryIds: Collection<String>?): Set<String>
fun findAllSeriesAndBookTags(filterOnLibraryIds: Collection<String>?): Set<String>
fun findAllSeriesAndBookTagsByLibrary(libraryId: String, filterOnLibraryIds: Collection<String>?): Set<String>
fun findAllSeriesAndBookTagsByCollection(collectionId: String, filterOnLibraryIds: Collection<String>?): Set<String>
fun findAllSeriesTags(filterOnLibraryIds: Collection<String>?): Set<String>
fun findAllSeriesTagsByLibrary(libraryId: String, filterOnLibraryIds: Collection<String>?): Set<String>
fun findAllSeriesTagsByCollection(collectionId: String, filterOnLibraryIds: Collection<String>?): Set<String>

12
komga/src/main/kotlin/org/gotson/komga/domain/service/MetadataAggregator.kt
View file

@ -9,11 +9,15 @@ class MetadataAggregator {
fun aggregate(metadatas: Collection<BookMetadata>): BookMetadataAggregation {
val authors = metadatas.flatMap { it.authors }.distinctBy { "${it.role}__${it.name}" }
val (summary, summaryNumber) = metadatas.sortedBy { it.numberSort }.find { it.summary.isNotBlank() }?.let {
it.summary to it.number
} ?: "" to ""
val tags = metadatas.flatMap { it.tags }.toSet()
val (summary, summaryNumber) = metadatas
.sortedBy { it.numberSort }
.find { it.summary.isNotBlank() }
?.let {
it.summary to it.number
} ?: ("" to "")
val releaseDate = metadatas.mapNotNull { it.releaseDate }.minOrNull()
return BookMetadataAggregation(authors = authors, releaseDate = releaseDate, summary = summary, summaryNumber = summaryNumber)
return BookMetadataAggregation(authors = authors, tags = tags, releaseDate = releaseDate, summary = summary, summaryNumber = summaryNumber)
}
}

36
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDao.kt
View file

@ -19,6 +19,7 @@ class BookMetadataAggregationDao(
private val d = Tables.BOOK_METADATA_AGGREGATION
private val a = Tables.BOOK_METADATA_AGGREGATION_AUTHOR
private val t = Tables.BOOK_METADATA_AGGREGATION_TAG
private val groupFields = arrayOf(*d.fields(), *a.fields())
@ -32,14 +33,23 @@ class BookMetadataAggregationDao(
dsl.select(*groupFields)
.from(d)
.leftJoin(a).on(d.SERIES_ID.eq(a.SERIES_ID))
.leftJoin(t).on(d.SERIES_ID.eq(t.SERIES_ID))
.where(d.SERIES_ID.`in`(seriesIds))
.groupBy(*groupFields)
.fetchGroups(
{ it.into(d) }, { it.into(a) }
).map { (dr, ar) ->
dr.toDomain(ar.filterNot { it.name == null }.map { it.toDomain() })
dr.toDomain(ar.filterNot { it.name == null }.map { it.toDomain() }, findTags(dr.seriesId))
}
private fun findTags(seriesId: String) =
dsl.select(t.TAG)
.from(t)
.where(t.SERIES_ID.eq(seriesId))
.fetchInto(t)
.mapNotNull { it.tag }
.toSet()
@Transactional
override fun insert(metadata: BookMetadataAggregation) {
dsl.insertInto(d)
@ -50,6 +60,7 @@ class BookMetadataAggregationDao(
.execute()
insertAuthors(metadata)
insertTags(metadata)
}
@Transactional
@ -66,7 +77,12 @@ class BookMetadataAggregationDao(
.where(a.SERIES_ID.eq(metadata.seriesId))
.execute()
dsl.deleteFrom(t)
.where(t.SERIES_ID.eq(metadata.seriesId))
.execute()
insertAuthors(metadata)
insertTags(metadata)
}
private fun insertAuthors(metadata: BookMetadataAggregation) {
@ -82,23 +98,39 @@ class BookMetadataAggregationDao(
}
}
private fun insertTags(metadata: BookMetadataAggregation) {
if (metadata.tags.isNotEmpty()) {
dsl.batch(
dsl.insertInto(t, t.SERIES_ID, t.TAG)
.values(null as String?, null)
).also { step ->
metadata.tags.forEach {
step.bind(metadata.seriesId, it)
}
}.execute()
}
}
@Transactional
override fun delete(seriesId: String) {
dsl.deleteFrom(a).where(a.SERIES_ID.eq(seriesId)).execute()
dsl.deleteFrom(t).where(t.SERIES_ID.eq(seriesId)).execute()
dsl.deleteFrom(d).where(d.SERIES_ID.eq(seriesId)).execute()
}
@Transactional
override fun delete(seriesIds: Collection<String>) {
dsl.deleteFrom(a).where(a.SERIES_ID.`in`(seriesIds)).execute()
dsl.deleteFrom(t).where(t.SERIES_ID.`in`(seriesIds)).execute()
dsl.deleteFrom(d).where(d.SERIES_ID.`in`(seriesIds)).execute()
}
override fun count(): Long = dsl.fetchCount(d).toLong()
private fun BookMetadataAggregationRecord.toDomain(authors: List<Author>) =
private fun BookMetadataAggregationRecord.toDomain(authors: List<Author>, tags: Set<String>) =
BookMetadataAggregation(
authors = authors,
tags = tags,
releaseDate = releaseDate,
summary = summary,
summaryNumber = summaryNumber,

51
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReferentialDao.kt
View file

@ -31,6 +31,7 @@ class ReferentialDao(
private val sd = Tables.SERIES_METADATA
private val bma = Tables.BOOK_METADATA_AGGREGATION
private val bmaa = Tables.BOOK_METADATA_AGGREGATION_AUTHOR
private val bmat = Tables.BOOK_METADATA_AGGREGATION_TAG
private val s = Tables.SERIES
private val b = Tables.BOOK
private val g = Tables.SERIES_METADATA_GENRE
@ -212,21 +213,47 @@ class ReferentialDao(
override fun findAllSeriesAndBookTags(filterOnLibraryIds: Collection<String>?): Set<String> =
dsl.select(bt.TAG.`as`("tag"))
.from(bt)
.apply {
filterOnLibraryIds?.let {
leftJoin(b).on(bt.BOOK_ID.eq(b.ID))
.where(b.LIBRARY_ID.`in`(it))
}
}
.apply { filterOnLibraryIds?.let { leftJoin(b).on(bt.BOOK_ID.eq(b.ID)).where(b.LIBRARY_ID.`in`(it)) } }
.union(
select(st.TAG.`as`("tag"))
.from(st)
.apply {
filterOnLibraryIds?.let {
leftJoin(s).on(st.SERIES_ID.eq(s.ID))
.where(s.LIBRARY_ID.`in`(it))
}
}
.apply { filterOnLibraryIds?.let { leftJoin(s).on(st.SERIES_ID.eq(s.ID)).where(s.LIBRARY_ID.`in`(it)) } }
)
.fetchSet(0, String::class.java)
.sortedBy { it.stripAccents().lowercase() }
.toSet()
override fun findAllSeriesAndBookTagsByLibrary(libraryId: String, filterOnLibraryIds: Collection<String>?): Set<String> =
dsl.select(bt.TAG.`as`("tag"))
.from(bt)
.leftJoin(b).on(bt.BOOK_ID.eq(b.ID))
.where(b.LIBRARY_ID.eq(libraryId))
.apply { filterOnLibraryIds?.let { and(b.LIBRARY_ID.`in`(it)) } }
.union(
select(st.TAG.`as`("tag"))
.from(st)
.leftJoin(s).on(st.SERIES_ID.eq(s.ID))
.where(s.LIBRARY_ID.eq(libraryId))
.apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } }
)
.fetchSet(0, String::class.java)
.sortedBy { it.stripAccents().lowercase() }
.toSet()
override fun findAllSeriesAndBookTagsByCollection(collectionId: String, filterOnLibraryIds: Collection<String>?): Set<String> =
dsl.select(bmat.TAG.`as`("tag"))
.from(bmat)
.leftJoin(s).on(bmat.SERIES_ID.eq(s.ID))
.leftJoin(cs).on(bmat.SERIES_ID.eq(cs.SERIES_ID))
.where(cs.COLLECTION_ID.eq(collectionId))
.apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } }
.union(
select(st.TAG.`as`("tag"))
.from(st)
.leftJoin(cs).on(st.SERIES_ID.eq(cs.SERIES_ID))
.leftJoin(s).on(st.SERIES_ID.eq(s.ID))
.where(cs.COLLECTION_ID.eq(collectionId))
.apply { filterOnLibraryIds?.let { and(s.LIBRARY_ID.`in`(it)) } }
)
.fetchSet(0, String::class.java)
.sortedBy { it.stripAccents().lowercase() }

34
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt
View file

@ -56,6 +56,7 @@ class SeriesDtoDao(
private val st = Tables.SERIES_METADATA_TAG
private val bma = Tables.BOOK_METADATA_AGGREGATION
private val bmaa = Tables.BOOK_METADATA_AGGREGATION_AUTHOR
private val bmat = Tables.BOOK_METADATA_AGGREGATION_TAG
private val fts = Tables.FTS_SERIES_METADATA
val countUnread: AggregateFunction<BigDecimal> = DSL.sum(DSL.`when`(r.COMPLETED.isNull, 1).otherwise(0))
@ -125,7 +126,11 @@ class SeriesDtoDao(
.leftJoin(bma).on(s.ID.eq(bma.SERIES_ID))
.leftJoin(rs).on(s.ID.eq(rs.SERIES_ID)).and(readProgressConditionSeries(userId))
.apply { if (joinConditions.genre) leftJoin(g).on(s.ID.eq(g.SERIES_ID)) }
.apply { if (joinConditions.tag) leftJoin(st).on(s.ID.eq(st.SERIES_ID)) }
.apply {
if (joinConditions.tag)
leftJoin(st).on(s.ID.eq(st.SERIES_ID))
.leftJoin(bmat).on(s.ID.eq(bmat.SERIES_ID))
}
.apply { if (joinConditions.collection) leftJoin(cs).on(s.ID.eq(cs.SERIES_ID)) }
.apply { if (joinConditions.aggregationAuthor) leftJoin(bmaa).on(s.ID.eq(bmaa.SERIES_ID)) }
.where(conditions)
@ -161,7 +166,11 @@ class SeriesDtoDao(
.leftJoin(bma).on(s.ID.eq(bma.SERIES_ID))
.leftJoin(rs).on(s.ID.eq(rs.SERIES_ID)).and(readProgressConditionSeries(userId))
.apply { if (joinConditions.genre) leftJoin(g).on(s.ID.eq(g.SERIES_ID)) }
.apply { if (joinConditions.tag) leftJoin(st).on(s.ID.eq(st.SERIES_ID)) }
.apply {
if (joinConditions.tag)
leftJoin(st).on(s.ID.eq(st.SERIES_ID))
.leftJoin(bmat).on(s.ID.eq(bmat.SERIES_ID))
}
.apply { if (joinConditions.collection) leftJoin(cs).on(s.ID.eq(cs.SERIES_ID)) }
.apply { if (joinConditions.aggregationAuthor) leftJoin(bmaa).on(s.ID.eq(bmaa.SERIES_ID)) }
@ -179,7 +188,11 @@ class SeriesDtoDao(
.leftJoin(bma).on(s.ID.eq(bma.SERIES_ID))
.leftJoin(rs).on(s.ID.eq(rs.SERIES_ID)).and(readProgressConditionSeries(userId))
.apply { if (joinConditions.genre) leftJoin(g).on(s.ID.eq(g.SERIES_ID)) }
.apply { if (joinConditions.tag) leftJoin(st).on(s.ID.eq(st.SERIES_ID)) }
.apply {
if (joinConditions.tag)
leftJoin(st).on(s.ID.eq(st.SERIES_ID))
.leftJoin(bmat).on(s.ID.eq(bmat.SERIES_ID))
}
.apply { if (joinConditions.collection) leftJoin(cs).on(s.ID.eq(cs.SERIES_ID)) }
.apply { if (joinConditions.aggregationAuthor) leftJoin(bmaa).on(s.ID.eq(bmaa.SERIES_ID)) }
.where(conditions)
@ -238,13 +251,17 @@ class SeriesDtoDao(
.filter { it.name != null }
.map { AuthorDto(it.name, it.role) }
val aggregatedTags = dsl.selectFrom(bmat)
.where(bmat.SERIES_ID.eq(sr.id))
.fetchSet(bmat.TAG)
sr.toDto(
sr.bookCount,
booksReadCount,
booksUnreadCount,
booksInProgressCount,
dr.toDto(genres, tags),
bmar.toDto(aggregatedAuthors)
bmar.toDto(aggregatedAuthors, aggregatedTags)
)
}
@ -261,12 +278,10 @@ class SeriesDtoDao(
if (deleted == false) c = c.and(s.DELETED_DATE.isNull)
if (!languages.isNullOrEmpty()) c = c.and(lower(d.LANGUAGE).`in`(languages.map { it.lowercase() }))
if (!genres.isNullOrEmpty()) c = c.and(lower(g.GENRE).`in`(genres.map { it.lowercase() }))
if (!tags.isNullOrEmpty()) c = c.and(lower(st.TAG).`in`(tags.map { it.lowercase() }))
if (!tags.isNullOrEmpty()) c = c.and(lower(st.TAG).`in`(tags.map { it.lowercase() }).or(lower(bmat.TAG).`in`(tags.map { it.lowercase() })))
if (!ageRatings.isNullOrEmpty()) {
val c1 = if (ageRatings.contains(null)) d.AGE_RATING.isNull else DSL.falseCondition()
val c2 = if (ageRatings.filterNotNull()
.isNotEmpty()
) d.AGE_RATING.`in`(ageRatings.filterNotNull()) else DSL.falseCondition()
val c2 = if (ageRatings.filterNotNull().isNotEmpty()) d.AGE_RATING.`in`(ageRatings.filterNotNull()) else DSL.falseCondition()
c = c.and(c1.or(c2))
}
// cast to String is necessary for SQLite, else the years in the IN block are coerced to Int, even though YEAR for SQLite uses strftime (string)
@ -370,9 +385,10 @@ class SeriesDtoDao(
totalBookCountLock = totalBookCountLock,
)
private fun BookMetadataAggregationRecord.toDto(authors: List<AuthorDto>) =
private fun BookMetadataAggregationRecord.toDto(authors: List<AuthorDto>, tags: Set<String>) =
BookMetadataAggregationDto(
authors = authors,
tags = tags,
releaseDate = releaseDate,
summary = summary,
summaryNumber = summaryNumber,

3
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/Utils.kt
View file

@ -5,7 +5,6 @@ import org.jooq.Condition
import org.jooq.Field
import org.jooq.SortField
import org.jooq.Table
import org.jooq.TableField
import org.jooq.impl.DSL
import org.springframework.data.domain.Sort
import org.sqlite.SQLiteException
@ -25,7 +24,7 @@ fun Sort.toOrderBy(sorts: Map<String, Field<out Any>>): List<SortField<out Any>>
fun LocalDateTime.toCurrentTimeZone(): LocalDateTime =
this.atZone(ZoneId.of("Z")).withZoneSameInstant(ZoneId.systemDefault()).toLocalDateTime()
fun TableField<*, String>.udfStripAccents() =
fun Field<String>.udfStripAccents() =
DSL.function(SqliteUdfDataSource.udfStripAccents, String::class.java, this)
fun Table<*>.match(term: String): Condition =

7
komga/src/main/kotlin/org/gotson/komga/interfaces/rest/ReferentialController.kt
View file

@ -93,15 +93,12 @@ class ReferentialController(
@GetMapping("v1/tags")
fun getTags(
@AuthenticationPrincipal principal: KomgaPrincipal,
// TODO: remove those parameters once Tachiyomi Extension is using the new /tags/series endpoint (changed in 0.87.4 - 21 Apr 2021)
@RequestParam(name = "library_id", required = false) libraryId: String?,
@RequestParam(name = "series_id", required = false) seriesId: String?,
@RequestParam(name = "collection_id", required = false) collectionId: String?
): Set<String> =
when {
libraryId != null -> referentialRepository.findAllSeriesTagsByLibrary(libraryId, principal.user.getAuthorizedLibraryIds(null))
seriesId != null -> referentialRepository.findAllBookTagsBySeries(seriesId, principal.user.getAuthorizedLibraryIds(null))
collectionId != null -> referentialRepository.findAllSeriesTagsByCollection(collectionId, principal.user.getAuthorizedLibraryIds(null))
libraryId != null -> referentialRepository.findAllSeriesAndBookTagsByLibrary(libraryId, principal.user.getAuthorizedLibraryIds(null))
collectionId != null -> referentialRepository.findAllSeriesAndBookTagsByCollection(collectionId, principal.user.getAuthorizedLibraryIds(null))
else -> referentialRepository.findAllSeriesAndBookTags(principal.user.getAuthorizedLibraryIds(null))
}

1
komga/src/main/kotlin/org/gotson/komga/interfaces/rest/dto/SeriesDto.kt
View file

@ -60,6 +60,7 @@ data class SeriesMetadataDto(
data class BookMetadataAggregationDto(
val authors: List<AuthorDto> = emptyList(),
val tags: Set<String> = emptySet(),
@JsonFormat(pattern = "yyyy-MM-dd")
val releaseDate: LocalDate?,
val summary: String,

12
komga/src/test/kotlin/org/gotson/komga/domain/service/MetadataAggregatorTest.kt
View file

@ -12,13 +12,14 @@ class MetadataAggregatorTest {
@Test
fun `given metadatas when aggregating then aggregation is relevant`() {
val metadatas = listOf(
BookMetadata(title = "ignored", summary = "summary 1", number = "1", numberSort = 1F, authors = listOf(Author("author1", "role1"), Author("author2", "role2")), releaseDate = LocalDate.of(2020, 1, 1)),
BookMetadata(title = "ignored", summary = "summary 2", number = "2", numberSort = 2F, authors = listOf(Author("author3", "role3"), Author("author2", "role3")), releaseDate = LocalDate.of(2021, 1, 1)),
BookMetadata(title = "ignored", summary = "summary 1", number = "1", numberSort = 1F, authors = listOf(Author("author1", "role1"), Author("author2", "role2")), releaseDate = LocalDate.of(2020, 1, 1), tags = setOf("tag1")),
BookMetadata(title = "ignored", summary = "summary 2", number = "2", numberSort = 2F, authors = listOf(Author("author3", "role3"), Author("author2", "role3")), releaseDate = LocalDate.of(2021, 1, 1), tags = setOf("tag2")),
)
val aggregation = aggregator.aggregate(metadatas)
assertThat(aggregation.authors).hasSize(4)
assertThat(aggregation.tags).hasSize(2)
assertThat(aggregation.releaseDate?.year).isEqualTo(2020)
assertThat(aggregation.summary).isEqualTo("summary 1")
assertThat(aggregation.summaryNumber).isEqualTo("1")
@ -50,14 +51,15 @@ class MetadataAggregatorTest {
}
@Test
fun `given metadatas with duplicate authors when aggregating then aggregation has no duplicate authors`() {
fun `given metadatas with duplicate authors or tags when aggregating then aggregation has no duplicates`() {
val metadatas = listOf(
BookMetadata(title = "ignored", number = "1", numberSort = 1F, authors = listOf(Author("author1", "role1"), Author("author2", "role2"))),
BookMetadata(title = "ignored", number = "2", numberSort = 2F, authors = listOf(Author("author1", "role1"), Author("author2", "role2"))),
BookMetadata(title = "ignored", number = "1", numberSort = 1F, authors = listOf(Author("author1", "role1"), Author("author2", "role2")), tags = setOf("tag1", "tag2")),
BookMetadata(title = "ignored", number = "2", numberSort = 2F, authors = listOf(Author("author1", "role1"), Author("author2", "role2")), tags = setOf("tag1")),
)
val aggregation = aggregator.aggregate(metadatas)
assertThat(aggregation.authors).hasSize(2)
assertThat(aggregation.tags).hasSize(2)
}
}

6
komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/BookMetadataAggregationDaoTest.kt
View file

@ -54,6 +54,7 @@ class BookMetadataAggregationDaoTest(
val now = LocalDateTime.now()
val metadata = BookMetadataAggregation(
authors = listOf(Author("author", "role")),
tags = setOf("tag1", "tag2"),
releaseDate = LocalDate.now(),
summary = "Summary",
summaryNumber = "1",
@ -74,6 +75,7 @@ class BookMetadataAggregationDaoTest(
assertThat(name).isEqualTo(metadata.authors.first().name)
assertThat(role).isEqualTo(metadata.authors.first().role)
}
assertThat(created.tags).containsExactlyInAnyOrderElementsOf(metadata.tags)
}
@Test
@ -104,6 +106,7 @@ class BookMetadataAggregationDaoTest(
val metadata = BookMetadataAggregation(
authors = listOf(Author("author", "role")),
tags = setOf("tag1", "tag2"),
releaseDate = LocalDate.now(),
summary = "Summary",
seriesId = series.id
@ -137,6 +140,7 @@ class BookMetadataAggregationDaoTest(
val metadata = BookMetadataAggregation(
authors = listOf(Author("author", "role")),
tags = setOf("tag1", "tag2"),
releaseDate = LocalDate.now(),
summary = "Summary",
summaryNumber = "1",
@ -153,6 +157,7 @@ class BookMetadataAggregationDaoTest(
summary = "SummaryUpdated",
summaryNumber = "2",
authors = listOf(Author("authorUpdated", "roleUpdated"), Author("author2", "role2")),
tags = setOf("tag1", "tag2updated"),
)
}
@ -170,5 +175,6 @@ class BookMetadataAggregationDaoTest(
assertThat(modified.authors).hasSize(2)
assertThat(modified.authors.map { it.name }).containsExactlyInAnyOrderElementsOf(updated.authors.map { it.name })
assertThat(modified.authors.map { it.role }).containsExactlyInAnyOrderElementsOf(updated.authors.map { it.role })
assertThat(modified.tags).containsExactlyInAnyOrderElementsOf(updated.tags)
}
}