Backup_Repos/komga
1
0
Fork 0
mirror of https://github.com/gotson/komga.git synced 2025-12-20 15:34:17 +01:00
Code Issues Projects Releases Wiki Activity

feat: import mylar metadata

Browse source 
closes #550
This commit is contained in:
Gauthier Roebroeck 2021-06-07 15:26:35 +08:00
parent 4bf2cd07b3
commit 528f676ce0
18 changed files with 326 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
komga/src/flyway/resources/db/migration/sqlite/V20210607144721__library_import_mylar_metadata.sql Normal file
View file

@ -0,0 +1,2 @@
alter table library
add column IMPORT_MYLAR_SERIES boolean NOT NULL DEFAULT 1;

1
komga/src/main/kotlin/org/gotson/komga/domain/model/Library.kt
View file

@ -16,6 +16,7 @@ data class Library(
val importComicInfoReadList: Boolean = true,
val importEpubBook: Boolean = true,
val importEpubSeries: Boolean = true,
val importMylarSeries: Boolean = true,
val importLocalArtwork: Boolean = true,
val importBarcodeIsbn: Boolean = true,
val scanForceModifiedTime: Boolean = false,

28
komga/src/main/kotlin/org/gotson/komga/domain/service/MetadataLifecycle.kt
View file

@ -20,10 +20,12 @@ import org.gotson.komga.domain.persistence.ReadListRepository
import org.gotson.komga.domain.persistence.SeriesCollectionRepository
import org.gotson.komga.domain.persistence.SeriesMetadataRepository
import org.gotson.komga.infrastructure.metadata.BookMetadataProvider
import org.gotson.komga.infrastructure.metadata.SeriesMetadataFromBookProvider
import org.gotson.komga.infrastructure.metadata.SeriesMetadataProvider
import org.gotson.komga.infrastructure.metadata.barcode.IsbnBarcodeProvider
import org.gotson.komga.infrastructure.metadata.comicrack.ComicInfoProvider
import org.gotson.komga.infrastructure.metadata.epub.EpubMetadataProvider
import org.gotson.komga.infrastructure.metadata.mylar.MylarSeriesProvider
import org.springframework.stereotype.Service
private val logger = KotlinLogging.logger {}
@ -31,6 +33,7 @@ private val logger = KotlinLogging.logger {}
@Service
class MetadataLifecycle(
private val bookMetadataProviders: List<BookMetadataProvider>,
private val seriesMetadataFromBookProviders: List<SeriesMetadataFromBookProvider>,
private val seriesMetadataProviders: List<SeriesMetadataProvider>,
private val metadataApplier: MetadataApplier,
private val metadataAggregator: MetadataAggregator,
@ -147,7 +150,7 @@ class MetadataLifecycle(
val library = libraryRepository.findById(series.libraryId)
var changed = false
seriesMetadataProviders.forEach { provider ->
seriesMetadataFromBookProviders.forEach { provider ->
when {
provider is ComicInfoProvider && !library.importComicInfoSeries && !library.importComicInfoCollection -> logger.info { "Library is not set to import series and collection metadata from ComicInfo, skipping" }
provider is EpubMetadataProvider && !library.importEpubSeries -> logger.info { "Library is not set to import series metadata from Epub, skipping" }
@ -171,6 +174,20 @@ class MetadataLifecycle(
}
}
seriesMetadataProviders.forEach { provider ->
when {
provider is MylarSeriesProvider && !library.importMylarSeries -> logger.info { "Library is not set to import series metadata from Mylar, skipping" }
else -> {
logger.debug { "Provider: $provider" }
val patch = provider.getSeriesMetadata(series)
if (provider is MylarSeriesProvider && library.importMylarSeries && patch != null) {
handlePatchForSeriesMetadata(patch, series)
}
}
}
}
if (changed) eventPublisher.publishEvent(DomainEvent.SeriesUpdated(series))
}
@ -219,11 +236,18 @@ class MetadataLifecycle(
collections = emptyList()
)
handlePatchForSeriesMetadata(aggregatedPatch, series)
}
private fun handlePatchForSeriesMetadata(
patch: SeriesMetadataPatch,
series: Series
) {
seriesMetadataRepository.findById(series.id).let {
logger.debug { "Apply metadata for series: $series" }
logger.debug { "Original metadata: $it" }
val patched = metadataApplier.apply(aggregatedPatch, it)
val patched = metadataApplier.apply(patch, it)
logger.debug { "Patched metadata: $patched" }
seriesMetadataRepository.update(patched)

3
komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDao.kt
View file

@ -67,6 +67,7 @@ class LibraryDao(
.set(l.IMPORT_COMICINFO_READLIST, library.importComicInfoReadList)
.set(l.IMPORT_EPUB_BOOK, library.importEpubBook)
.set(l.IMPORT_EPUB_SERIES, library.importEpubSeries)
.set(l.IMPORT_MYLAR_SERIES, library.importMylarSeries)
.set(l.IMPORT_LOCAL_ARTWORK, library.importLocalArtwork)
.set(l.IMPORT_BARCODE_ISBN, library.importBarcodeIsbn)
.set(l.SCAN_FORCE_MODIFIED_TIME, library.scanForceModifiedTime)
@ -88,6 +89,7 @@ class LibraryDao(
.set(l.IMPORT_COMICINFO_READLIST, library.importComicInfoReadList)
.set(l.IMPORT_EPUB_BOOK, library.importEpubBook)
.set(l.IMPORT_EPUB_SERIES, library.importEpubSeries)
.set(l.IMPORT_MYLAR_SERIES, library.importMylarSeries)
.set(l.IMPORT_LOCAL_ARTWORK, library.importLocalArtwork)
.set(l.IMPORT_BARCODE_ISBN, library.importBarcodeIsbn)
.set(l.SCAN_FORCE_MODIFIED_TIME, library.scanForceModifiedTime)
@ -112,6 +114,7 @@ class LibraryDao(
importComicInfoReadList = importComicinfoReadlist,
importEpubBook = importEpubBook,
importEpubSeries = importEpubSeries,
importMylarSeries = importMylarSeries,
importLocalArtwork = importLocalArtwork,
importBarcodeIsbn = importBarcodeIsbn,
scanForceModifiedTime = scanForceModifiedTime,

8
komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/SeriesMetadataFromBookProvider.kt Normal file
View file

@ -0,0 +1,8 @@
package org.gotson.komga.infrastructure.metadata
import org.gotson.komga.domain.model.BookWithMedia
import org.gotson.komga.domain.model.SeriesMetadataPatch
interface SeriesMetadataFromBookProvider {
fun getSeriesMetadataFromBook(book: BookWithMedia): SeriesMetadataPatch?
}

4
komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/SeriesMetadataProvider.kt
View file

@ -1,8 +1,8 @@
package org.gotson.komga.infrastructure.metadata
import org.gotson.komga.domain.model.BookWithMedia
import org.gotson.komga.domain.model.Series
import org.gotson.komga.domain.model.SeriesMetadataPatch
interface SeriesMetadataProvider {
fun getSeriesMetadataFromBook(book: BookWithMedia): SeriesMetadataPatch?
fun getSeriesMetadata(series: Series): SeriesMetadataPatch?
}

6
komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/comicrack/ComicInfoProvider.kt
View file

@ -10,7 +10,7 @@ import org.gotson.komga.domain.model.SeriesMetadata
import org.gotson.komga.domain.model.SeriesMetadataPatch
import org.gotson.komga.domain.service.BookAnalyzer
import org.gotson.komga.infrastructure.metadata.BookMetadataProvider
import org.gotson.komga.infrastructure.metadata.SeriesMetadataProvider
import org.gotson.komga.infrastructure.metadata.SeriesMetadataFromBookProvider
import org.gotson.komga.infrastructure.metadata.comicrack.dto.ComicInfo
import org.gotson.komga.infrastructure.metadata.comicrack.dto.Manga
import org.gotson.komga.infrastructure.validation.BCP47TagValidator
@ -26,7 +26,7 @@ private const val COMIC_INFO = "ComicInfo.xml"
class ComicInfoProvider(
@Autowired(required = false) private val mapper: XmlMapper = XmlMapper(),
private val bookAnalyzer: BookAnalyzer
) : BookMetadataProvider, SeriesMetadataProvider {
) : BookMetadataProvider, SeriesMetadataFromBookProvider {
override fun getCapabilities(): List<BookMetadataPatchCapability> =
listOf(
@ -119,7 +119,7 @@ class ComicInfoProvider(
val fileContent = bookAnalyzer.getFileContent(book, COMIC_INFO)
return mapper.readValue(fileContent, ComicInfo::class.java)
} catch (e: Exception) {
logger.error(e) { "Error while retrieving metadata from ComicInfo.xml" }
logger.error(e) { "Error while retrieving metadata from $COMIC_INFO" }
return null
}
}

4
komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/epub/EpubMetadataProvider.kt
View file

@ -9,7 +9,7 @@ import org.gotson.komga.domain.model.SeriesMetadata
import org.gotson.komga.domain.model.SeriesMetadataPatch
import org.gotson.komga.infrastructure.mediacontainer.EpubExtractor
import org.gotson.komga.infrastructure.metadata.BookMetadataProvider
import org.gotson.komga.infrastructure.metadata.SeriesMetadataProvider
import org.gotson.komga.infrastructure.metadata.SeriesMetadataFromBookProvider
import org.gotson.komga.infrastructure.validation.BCP47TagValidator
import org.jsoup.Jsoup
import org.jsoup.parser.Parser
@ -22,7 +22,7 @@ import java.time.format.DateTimeFormatter
class EpubMetadataProvider(
private val epubExtractor: EpubExtractor,
private val isbnValidator: ISBNValidator
) : BookMetadataProvider, SeriesMetadataProvider {
) : BookMetadataProvider, SeriesMetadataFromBookProvider {
private val relators = mapOf(
"aut" to "writer",

52
komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/mylar/MylarSeriesProvider.kt Normal file
View file

@ -0,0 +1,52 @@
package org.gotson.komga.infrastructure.metadata.mylar
import com.fasterxml.jackson.databind.ObjectMapper
import mu.KotlinLogging
import org.gotson.komga.domain.model.Series
import org.gotson.komga.domain.model.SeriesMetadata
import org.gotson.komga.domain.model.SeriesMetadataPatch
import org.gotson.komga.infrastructure.metadata.SeriesMetadataProvider
import org.gotson.komga.infrastructure.metadata.mylar.dto.Status
import org.springframework.stereotype.Service
import kotlin.io.path.notExists
import org.gotson.komga.infrastructure.metadata.mylar.dto.Series as MylarSeries
private val logger = KotlinLogging.logger {}
private const val SERIES_JSON = "series.json"
@Service
class MylarSeriesProvider(
private val mapper: ObjectMapper,
) : SeriesMetadataProvider {
override fun getSeriesMetadata(series: Series): SeriesMetadataPatch? {
try {
val seriesJsonPath = series.path.resolve(SERIES_JSON)
if (seriesJsonPath.notExists()) {
logger.debug { "Series folder does not contain any $SERIES_JSON file: $series" }
return null
}
val metadata = mapper.readValue(seriesJsonPath.toFile(), MylarSeries::class.java).metadata.first()
return SeriesMetadataPatch(
title = metadata.name,
titleSort = metadata.name,
status = when (metadata.status) {
Status.Ended -> SeriesMetadata.Status.ENDED
Status.Continuing -> SeriesMetadata.Status.ONGOING
},
summary = metadata.descriptionFormatted ?: metadata.descriptionText,
readingDirection = null,
publisher = metadata.publisher,
ageRating = null,
language = null,
genres = null,
collections = emptyList(),
)
} catch (e: Exception) {
logger.error(e) { "Error while retrieving metadata from $SERIES_JSON" }
return null
}
}
}

42
komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/mylar/dto/MylarMetadata.kt Normal file
View file

@ -0,0 +1,42 @@
package org.gotson.komga.infrastructure.metadata.mylar.dto
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.annotation.JsonProperty
@JsonIgnoreProperties(ignoreUnknown = true)
data class MylarMetadata(
val type: String,
val publisher: String,
val imprint: String?,
val name: String,
@field:JsonProperty("comicid")
val comicId: String,
val year: Int,
@field:JsonProperty("description_text")
val descriptionText: String?,
@field:JsonProperty("description_formatted")
val descriptionFormatted: String?,
val volume: String?,
@field:JsonProperty("booktype")
val bookType: String,
@field:JsonProperty("ComicImage")
val comicImage: String,
@field:JsonProperty("total_issues")
val totalIssues: Int,
@field:JsonProperty("publication_run")
val publicationRun: String,
val status: Status
)

8
komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/mylar/dto/Series.kt Normal file
View file

@ -0,0 +1,8 @@
package org.gotson.komga.infrastructure.metadata.mylar.dto
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
@JsonIgnoreProperties(ignoreUnknown = true)
data class Series(
val metadata: List<MylarMetadata>
)

5
komga/src/main/kotlin/org/gotson/komga/infrastructure/metadata/mylar/dto/Status.kt Normal file
View file

@ -0,0 +1,5 @@
package org.gotson.komga.infrastructure.metadata.mylar.dto
enum class Status {
Ended, Continuing
}

6
komga/src/main/kotlin/org/gotson/komga/interfaces/rest/LibraryController.kt
View file

@ -82,6 +82,7 @@ class LibraryController(
importComicInfoReadList = library.importComicInfoReadList,
importEpubBook = library.importEpubBook,
importEpubSeries = library.importEpubSeries,
importMylarSeries = library.importMylarSeries,
importLocalArtwork = library.importLocalArtwork,
importBarcodeIsbn = library.importBarcodeIsbn,
scanForceModifiedTime = library.scanForceModifiedTime,
@ -120,6 +121,7 @@ class LibraryController(
importComicInfoReadList = library.importComicInfoReadList,
importEpubBook = library.importEpubBook,
importEpubSeries = library.importEpubSeries,
importMylarSeries = library.importMylarSeries,
importLocalArtwork = library.importLocalArtwork,
importBarcodeIsbn = library.importBarcodeIsbn,
scanForceModifiedTime = library.scanForceModifiedTime,
@ -191,6 +193,7 @@ data class LibraryCreationDto(
val importComicInfoReadList: Boolean = true,
val importEpubBook: Boolean = true,
val importEpubSeries: Boolean = true,
val importMylarSeries: Boolean = true,
val importLocalArtwork: Boolean = true,
val importBarcodeIsbn: Boolean = true,
val scanForceModifiedTime: Boolean = false,
@ -210,6 +213,7 @@ data class LibraryDto(
val importComicInfoReadList: Boolean,
val importEpubBook: Boolean,
val importEpubSeries: Boolean,
val importMylarSeries: Boolean,
val importLocalArtwork: Boolean,
val importBarcodeIsbn: Boolean,
val scanForceModifiedTime: Boolean,
@ -228,6 +232,7 @@ data class LibraryUpdateDto(
val importComicInfoReadList: Boolean,
val importEpubBook: Boolean,
val importEpubSeries: Boolean,
val importMylarSeries: Boolean,
val importLocalArtwork: Boolean,
val importBarcodeIsbn: Boolean,
val scanForceModifiedTime: Boolean,
@ -247,6 +252,7 @@ fun Library.toDto(includeRoot: Boolean) = LibraryDto(
importComicInfoReadList = importComicInfoReadList,
importEpubBook = importEpubBook,
importEpubSeries = importEpubSeries,
importMylarSeries = importMylarSeries,
importLocalArtwork = importLocalArtwork,
importBarcodeIsbn = importBarcodeIsbn,
scanForceModifiedTime = scanForceModifiedTime,

2
komga/src/test/kotlin/org/gotson/komga/infrastructure/jooq/LibraryDaoTest.kt
View file

@ -61,6 +61,7 @@ class LibraryDaoTest(
importComicInfoSeries = false,
importComicInfoBook = false,
importComicInfoReadList = false,
importMylarSeries = false,
importBarcodeIsbn = false,
importLocalArtwork = false,
repairExtensions = true,
@ -88,6 +89,7 @@ class LibraryDaoTest(
assertThat(modified.importComicInfoReadList).isEqualTo(updated.importComicInfoReadList)
assertThat(modified.importBarcodeIsbn).isEqualTo(updated.importBarcodeIsbn)
assertThat(modified.importLocalArtwork).isEqualTo(updated.importLocalArtwork)
assertThat(modified.importMylarSeries).isEqualTo(updated.importMylarSeries)
assertThat(modified.repairExtensions).isEqualTo(updated.repairExtensions)
assertThat(modified.convertToCbz).isEqualTo(updated.convertToCbz)
assertThat(modified.emptyTrashAfterScan).isEqualTo(updated.emptyTrashAfterScan)

61
komga/src/test/kotlin/org/gotson/komga/infrastructure/metadata/mylar/MylarSeriesProviderTest.kt Normal file
View file

@ -0,0 +1,61 @@
package org.gotson.komga.infrastructure.metadata.mylar
import com.fasterxml.jackson.databind.ObjectMapper
import io.mockk.every
import io.mockk.mockk
import org.assertj.core.api.Assertions.assertThat
import org.gotson.komga.domain.model.SeriesMetadata
import org.gotson.komga.domain.model.makeSeries
import org.gotson.komga.infrastructure.metadata.mylar.dto.MylarMetadata
import org.gotson.komga.infrastructure.metadata.mylar.dto.Series
import org.gotson.komga.infrastructure.metadata.mylar.dto.Status
import org.junit.jupiter.api.Test
import org.springframework.core.io.ClassPathResource
import java.io.File
class MylarSeriesProviderTest {
private val mockMapper = mockk<ObjectMapper>()
private val mylarSeriesProvider = MylarSeriesProvider(mockMapper)
private val series = makeSeries("series", url = ClassPathResource("mylar").url)
@Test
fun `given seriesJson when getting series metadata then metadata patch is valid`() {
val metadata = MylarMetadata(
type = "comicSeries",
publisher = "DC",
imprint = "Vertigo",
name = "Sandman",
comicId = "12345",
year = 1990,
descriptionText = "Sandman comics",
descriptionFormatted = "Sandman comics formatted",
volume = "1",
bookType = "TPB",
comicImage = "unused",
totalIssues = 2,
publicationRun = "unused",
status = Status.Ended,
)
val root = Series(listOf(metadata))
every { mockMapper.readValue(any<File>(), Series::class.java) } returns root
val patch = mylarSeriesProvider.getSeriesMetadata(series)!!
with(patch) {
assertThat(title).isEqualTo("Sandman")
assertThat(titleSort).isEqualTo("Sandman")
assertThat(status).isEqualTo(SeriesMetadata.Status.ENDED)
assertThat(summary).isEqualTo("Sandman comics formatted")
assertThat(readingDirection).isNull()
assertThat(publisher).isEqualTo("DC")
assertThat(ageRating).isNull()
assertThat(language).isNull()
assertThat(genres).isNull()
assertThat(collections).isEmpty()
}
}
}

61
komga/src/test/kotlin/org/gotson/komga/infrastructure/metadata/mylar/dto/SeriesTest.kt Normal file
View file

@ -0,0 +1,61 @@
package org.gotson.komga.infrastructure.metadata.mylar.dto
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import com.fasterxml.jackson.module.kotlin.registerKotlinModule
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.springframework.core.io.ClassPathResource
class SeriesTest {
private val mapper = ObjectMapper().registerKotlinModule()
@Test
fun `given valid json file when deserializing then properties are available`() {
val file = ClassPathResource("mylar/series.json")
val seriesJson = mapper.readValue<Series>(file.url)
assertThat(seriesJson.metadata).hasSize(1)
with(seriesJson.metadata.first()) {
assertThat(type).isEqualTo("comicSeries")
assertThat(publisher).isEqualTo("DC Comics")
assertThat(imprint).isNull()
assertThat(name).isEqualTo("American Vampire 1976")
assertThat(comicId).isEqualTo("130865")
assertThat(year).isEqualTo(2020)
assertThat(descriptionText).isEqualTo("Nine issue mini-series, the closing chapter of American Vampire")
assertThat(descriptionFormatted).isEqualTo("Nine issue mini-series, the closing chapter of American Vampire")
assertThat(volume).isNull()
assertThat(bookType).isEqualTo("Print")
assertThat(comicImage).isEqualTo("https://comicvine.gamespot.com/a/uploads/scale_large/6/67663/7603293-01.jpg")
assertThat(totalIssues).isEqualTo(8)
assertThat(publicationRun).isEqualTo("December 2020 - Present")
assertThat(status).isEqualTo(Status.Continuing)
}
}
@Test
fun `given another valid json file when deserializing then properties are available`() {
val file = ClassPathResource("mylar/series1.json")
val seriesJson = mapper.readValue<Series>(file.url)
assertThat(seriesJson.metadata).hasSize(1)
with(seriesJson.metadata.first()) {
assertThat(type).isEqualTo("comicSeries")
assertThat(publisher).isEqualTo("IDW Publishing")
assertThat(imprint).isNull()
assertThat(name).isEqualTo("Usagi Yojimbo")
assertThat(comicId).isEqualTo("119731")
assertThat(year).isEqualTo(2019)
assertThat(descriptionText).isNull()
assertThat(descriptionFormatted).isNull()
assertThat(volume).isEqualTo("v4")
assertThat(bookType).isEqualTo("Print")
assertThat(comicImage).isEqualTo("https://comicvine1.cbsistatic.com/uploads/scale_large/6/67663/6974029-01a.jpg")
assertThat(totalIssues).isEqualTo(19)
assertThat(publicationRun).isEqualTo("June 2019 - Present")
assertThat(status).isEqualTo(Status.Ended)
}
}
}

21
komga/src/test/resources/mylar/series.json Executable file
View file

@ -0,0 +1,21 @@
{
"metadata": [
{
"type": "comicSeries",
"publisher": "DC Comics",
"imprint": null,
"name": "American Vampire 1976",
"comicid": "130865",
"year": "2020",
"description_text": "Nine issue mini-series, the closing chapter of American Vampire",
"description_formatted": "Nine issue mini-series, the closing chapter of American Vampire",
"volume": null,
"booktype": "Print",
"collects": null,
"ComicImage": "https://comicvine.gamespot.com/a/uploads/scale_large/6/67663/7603293-01.jpg",
"total_issues": 8,
"publication_run": "December 2020 - Present",
"status": "Continuing"
}
]
}

21
komga/src/test/resources/mylar/series1.json Executable file
View file

@ -0,0 +1,21 @@
{
"metadata": [
{
"type": "comicSeries",
"publisher": "IDW Publishing",
"imprint": null,
"name": "Usagi Yojimbo",
"comicid": "119731",
"year": "2019",
"description_text": null,
"description_formatted": null,
"volume": "v4",
"booktype": "Print",
"collects": null,
"ComicImage": "https://comicvine1.cbsistatic.com/uploads/scale_large/6/67663/6974029-01a.jpg",
"total_issues": 19,
"publication_run": "June 2019 - Present",
"status": "Ended"
}
]
}