diff --git a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt index 549b5bc1d..085d87345 100644 --- a/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt +++ b/komga/src/main/kotlin/org/gotson/komga/application/tasks/TaskHandler.kt @@ -1,5 +1,6 @@ package org.gotson.komga.application.tasks +import io.micrometer.core.instrument.MeterRegistry import mu.KotlinLogging import org.gotson.komga.domain.persistence.BookRepository import org.gotson.komga.domain.persistence.LibraryRepository @@ -16,10 +17,13 @@ import org.gotson.komga.domain.service.SeriesMetadataLifecycle import org.gotson.komga.infrastructure.jms.QUEUE_FACTORY import org.gotson.komga.infrastructure.jms.QUEUE_TASKS import org.gotson.komga.infrastructure.search.SearchIndexLifecycle +import org.gotson.komga.interfaces.scheduler.METER_TASKS_EXECUTION +import org.gotson.komga.interfaces.scheduler.METER_TASKS_FAILURE import org.springframework.jms.annotation.JmsListener import org.springframework.stereotype.Service import java.nio.file.Paths import kotlin.time.measureTime +import kotlin.time.toJavaDuration private val logger = KotlinLogging.logger {} @@ -39,6 +43,7 @@ class TaskHandler( private val bookConverter: BookConverter, private val bookPageEditor: BookPageEditor, private val searchIndexLifecycle: SearchIndexLifecycle, + private val meterRegistry: MeterRegistry, ) { @JmsListener(destination = QUEUE_TASKS, containerFactory = QUEUE_FACTORY, concurrency = "#{@komgaProperties.taskConsumers}-#{@komgaProperties.taskConsumersMax}") @@ -151,9 +156,11 @@ class TaskHandler( } }.also { logger.info { "Task $task executed in $it" } + meterRegistry.timer(METER_TASKS_EXECUTION, "type", task.javaClass.simpleName).record(it.toJavaDuration()) } } catch (e: Exception) { logger.error(e) { "Task $task execution failed" } + meterRegistry.counter(METER_TASKS_FAILURE, "type", task.javaClass.simpleName).increment() } } } diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/DomainEvent.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/DomainEvent.kt index ac647118e..6e9bb2469 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/DomainEvent.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/DomainEvent.kt @@ -8,6 +8,7 @@ sealed class DomainEvent : Serializable { data class LibraryAdded(val library: Library) : DomainEvent() data class LibraryUpdated(val library: Library) : DomainEvent() data class LibraryDeleted(val library: Library) : DomainEvent() + data class LibraryScanned(val library: Library) : DomainEvent() data class SeriesAdded(val series: Series) : DomainEvent() data class SeriesUpdated(val series: Series) : DomainEvent() diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt index 4bd96ca95..08e2a3921 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/BookRepository.kt @@ -5,6 +5,7 @@ import org.gotson.komga.domain.model.BookSearch import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort +import java.math.BigDecimal import java.net.URL interface BookRepository { @@ -43,4 +44,7 @@ interface BookRepository { fun deleteAll() fun count(): Long + fun countGroupedByLibraryName(): Map + + fun getFilesizeGroupedByLibraryName(): Map } diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/ReadListRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/ReadListRepository.kt index 6f321673c..f5a8f1b54 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/ReadListRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/ReadListRepository.kt @@ -42,4 +42,6 @@ interface ReadListRepository { fun deleteAll() fun existsByName(name: String): Boolean + + fun count(): Long } diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesCollectionRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesCollectionRepository.kt index c74dfccac..7107d73bc 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesCollectionRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesCollectionRepository.kt @@ -42,4 +42,6 @@ interface SeriesCollectionRepository { fun deleteAll() fun existsByName(name: String): Boolean + + fun count(): Long } diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesRepository.kt index 9fd8f5277..e49e65179 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SeriesRepository.kt @@ -26,4 +26,5 @@ interface SeriesRepository { fun deleteAll() fun count(): Long + fun countGroupedByLibraryName(): Map } diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SidecarRepository.kt b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SidecarRepository.kt index 1e4016462..6e93be1ef 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SidecarRepository.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/persistence/SidecarRepository.kt @@ -11,4 +11,6 @@ interface SidecarRepository { fun deleteByLibraryIdAndUrls(libraryId: String, urls: Collection) fun deleteByLibraryId(libraryId: String) + + fun countGroupedByLibraryName(): Map } diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt index 4fe181896..0b59917c9 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryContentLifecycle.kt @@ -231,6 +231,8 @@ class LibraryContentLifecycle( if (library.emptyTrashAfterScan) emptyTrash(library) else cleanupEmptySets() }.also { logger.info { "Library updated in $it" } } + + eventPublisher.publishEvent(DomainEvent.LibraryScanned(library)) } /** diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt index 5293f58d4..7af4dece0 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDao.kt @@ -16,6 +16,7 @@ import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional +import java.math.BigDecimal import java.net.URL import java.time.LocalDateTime import java.time.ZoneId @@ -29,6 +30,7 @@ class BookDao( private val m = Tables.MEDIA private val d = Tables.BOOK_METADATA private val r = Tables.READ_PROGRESS + private val l = Tables.LIBRARY private val sorts = mapOf( "createdDate" to b.CREATED_DATE, @@ -314,6 +316,20 @@ class BookDao( override fun count(): Long = dsl.fetchCount(b).toLong() + override fun countGroupedByLibraryName(): Map = + dsl.select(l.NAME, DSL.count(b.ID)) + .from(l) + .leftJoin(b).on(l.ID.eq(b.LIBRARY_ID)) + .groupBy(l.NAME) + .fetchMap(l.NAME, DSL.count(b.ID)) + + override fun getFilesizeGroupedByLibraryName(): Map = + dsl.select(l.NAME, DSL.sum(b.FILE_SIZE)) + .from(l) + .leftJoin(b).on(l.ID.eq(b.LIBRARY_ID)) + .groupBy(l.NAME) + .fetchMap(l.NAME, DSL.sum(b.FILE_SIZE)) + private fun BookSearch.toCondition(): Condition { var c: Condition = DSL.trueCondition() diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt index eb7117a50..de6206c7d 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/ReadListDao.kt @@ -247,6 +247,8 @@ class ReadListDao( .where(rl.NAME.equalIgnoreCase(name)), ) + override fun count(): Long = dsl.fetchCount(rl).toLong() + private fun ReadlistRecord.toDomain(bookIds: SortedMap) = ReadList( name = name, diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt index d6bad86ef..e94bdf7cd 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesCollectionDao.kt @@ -247,6 +247,8 @@ class SeriesCollectionDao( .where(c.NAME.equalIgnoreCase(name)), ) + override fun count(): Long = dsl.fetchCount(c).toLong() + private fun CollectionRecord.toDomain(seriesIds: List) = SeriesCollection( name = name, diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt index 70044f209..89e3cafb1 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDao.kt @@ -24,6 +24,7 @@ class SeriesDao( private val s = Tables.SERIES private val d = Tables.SERIES_METADATA private val cs = Tables.COLLECTION_SERIES + private val l = Tables.LIBRARY override fun findAll(): Collection = dsl.selectFrom(s) @@ -136,6 +137,13 @@ class SeriesDao( override fun count(): Long = dsl.fetchCount(s).toLong() + override fun countGroupedByLibraryName(): Map = + dsl.select(l.NAME, DSL.count(s.ID)) + .from(l) + .leftJoin(s).on(l.ID.eq(s.LIBRARY_ID)) + .groupBy(l.NAME) + .fetchMap(l.NAME, DSL.count(s.ID)) + private fun SeriesSearch.toCondition(): Condition { var c: Condition = DSL.trueCondition() diff --git a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt index bdcefdbf5..ff68c533c 100644 --- a/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt +++ b/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SidecarDao.kt @@ -6,6 +6,7 @@ import org.gotson.komga.domain.persistence.SidecarRepository import org.gotson.komga.jooq.Tables import org.gotson.komga.jooq.tables.records.SidecarRecord import org.jooq.DSLContext +import org.jooq.impl.DSL import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional @@ -17,6 +18,7 @@ class SidecarDao( @Value("#{@komgaProperties.database.batchChunkSize}") private val batchSize: Int, ) : SidecarRepository { private val sc = Tables.SIDECAR + private val l = Tables.LIBRARY override fun findAll(): Collection = dsl.selectFrom(sc).fetch().map { it.toDomain() } @@ -52,6 +54,13 @@ class SidecarDao( .execute() } + override fun countGroupedByLibraryName(): Map = + dsl.select(l.NAME, DSL.count(sc.URL)) + .from(l) + .leftJoin(sc).on(l.ID.eq(sc.LIBRARY_ID)) + .groupBy(l.NAME) + .fetchMap(l.NAME, DSL.count(sc.URL)) + private fun SidecarRecord.toDomain() = SidecarStored( url = URL(url), diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/MetricsPublisherController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/MetricsPublisherController.kt new file mode 100644 index 000000000..518802257 --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/MetricsPublisherController.kt @@ -0,0 +1,121 @@ +package org.gotson.komga.interfaces.scheduler + +import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.MultiGauge +import io.micrometer.core.instrument.MultiGauge.Row +import io.micrometer.core.instrument.Tags +import io.micrometer.core.instrument.Timer +import mu.KotlinLogging +import org.gotson.komga.domain.model.DomainEvent +import org.gotson.komga.domain.persistence.BookRepository +import org.gotson.komga.domain.persistence.LibraryRepository +import org.gotson.komga.domain.persistence.ReadListRepository +import org.gotson.komga.domain.persistence.SeriesCollectionRepository +import org.gotson.komga.domain.persistence.SeriesRepository +import org.gotson.komga.domain.persistence.SidecarRepository +import org.gotson.komga.infrastructure.jms.TOPIC_EVENTS +import org.gotson.komga.infrastructure.jms.TOPIC_FACTORY +import org.springframework.boot.context.event.ApplicationReadyEvent +import org.springframework.context.event.EventListener +import org.springframework.jms.annotation.JmsListener +import org.springframework.stereotype.Component +import java.util.concurrent.atomic.AtomicLong + +private val logger = KotlinLogging.logger {} + +private const val LIBRARIES = "libraries" +private const val SERIES = "series" +private const val BOOKS = "books" +private const val BOOKS_FILESIZE = "books.filesize" +private const val COLLECTIONS = "collections" +private const val READLISTS = "readlists" +private const val SIDECARS = "sidecars" + +const val METER_TASKS_EXECUTION = "komga.tasks.execution" +const val METER_TASKS_FAILURE = "komga.tasks.failure" + +@Component +class MetricsPublisherController( + private val libraryRepository: LibraryRepository, + private val bookRepository: BookRepository, + private val seriesRepository: SeriesRepository, + private val collectionRepository: SeriesCollectionRepository, + private val readListRepository: ReadListRepository, + private val sidecarRepository: SidecarRepository, + private val meterRegistry: MeterRegistry, +) { + + init { + Timer.builder(METER_TASKS_EXECUTION) + .description("Task execution time") + .register(meterRegistry) + + Counter.builder(METER_TASKS_FAILURE) + .description("Count of failed tasks") + .register(meterRegistry) + } + + private final val entitiesMultiTag = listOf(SERIES, BOOKS, BOOKS_FILESIZE, SIDECARS) + private final val entitiesNoTags = listOf(LIBRARIES, COLLECTIONS, READLISTS) + private final val allEntities = entitiesMultiTag + entitiesNoTags + + val multiGauges = entitiesMultiTag.associateWith { entity -> + MultiGauge.builder("komga.$entity") + .description("The number of $entity") + .baseUnit("count") + .register(meterRegistry) + } + + val noTagGauges = entitiesNoTags.associateWith { entity -> + AtomicLong(0).also { value -> + Gauge.builder("komga.$entity", value) { value.get().toDouble() } + .description("The number of $entity") + .baseUnit("count") + .register(meterRegistry) + } + } + + val bookFileSizeGauge = MultiGauge.builder("komga.$BOOKS_FILESIZE") + .description("The cumulated filesize of books") + .baseUnit("bytes") + .register(meterRegistry) + + @JmsListener(destination = TOPIC_EVENTS, containerFactory = TOPIC_FACTORY) + private fun pushMetricsOnEvent(event: DomainEvent) { + when (event) { + is DomainEvent.LibraryScanned -> entitiesMultiTag.forEach { pushMetricsCount(it) } + + is DomainEvent.LibraryAdded -> noTagGauges[LIBRARIES]?.incrementAndGet() + is DomainEvent.LibraryDeleted -> { + noTagGauges[LIBRARIES]?.decrementAndGet() + entitiesMultiTag.forEach { pushMetricsCount(it) } + } + is DomainEvent.CollectionAdded -> noTagGauges[COLLECTIONS]?.incrementAndGet() + is DomainEvent.CollectionDeleted -> noTagGauges[COLLECTIONS]?.decrementAndGet() + is DomainEvent.ReadListAdded -> noTagGauges[READLISTS]?.incrementAndGet() + is DomainEvent.ReadListDeleted -> noTagGauges[READLISTS]?.decrementAndGet() + + else -> Unit + } + } + + @EventListener(ApplicationReadyEvent::class) + fun pushAllMetrics() { + allEntities.forEach { pushMetricsCount(it) } + } + + private fun pushMetricsCount(entity: String) { + when (entity) { + LIBRARIES -> noTagGauges[LIBRARIES]?.set(libraryRepository.count()) + COLLECTIONS -> noTagGauges[COLLECTIONS]?.set(collectionRepository.count()) + READLISTS -> noTagGauges[READLISTS]?.set(readListRepository.count()) + + SERIES -> multiGauges[SERIES]?.register(seriesRepository.countGroupedByLibraryName().map { Row.of(Tags.of("library", it.key), it.value) }) + BOOKS -> multiGauges[BOOKS]?.register(bookRepository.countGroupedByLibraryName().map { Row.of(Tags.of("library", it.key), it.value) }) + BOOKS_FILESIZE -> bookFileSizeGauge.register(bookRepository.getFilesizeGroupedByLibraryName().map { Row.of(Tags.of("library", it.key), it.value) }) + SIDECARS -> multiGauges[SIDECARS]?.register(sidecarRepository.countGroupedByLibraryName().map { Row.of(Tags.of("library", it.key), it.value) }) + } + } +} diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/sse/SseController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/sse/SseController.kt index 5a03b2ae1..ae53c9fe6 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/sse/SseController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/sse/SseController.kt @@ -77,6 +77,7 @@ class SseController( is DomainEvent.LibraryAdded -> emitSse("LibraryAdded", LibrarySseDto(event.library.id)) is DomainEvent.LibraryUpdated -> emitSse("LibraryChanged", LibrarySseDto(event.library.id)) is DomainEvent.LibraryDeleted -> emitSse("LibraryDeleted", LibrarySseDto(event.library.id)) + is DomainEvent.LibraryScanned -> Unit is DomainEvent.SeriesAdded -> emitSse("SeriesAdded", SeriesSseDto(event.series.id, event.series.libraryId)) is DomainEvent.SeriesUpdated -> emitSse("SeriesChanged", SeriesSseDto(event.series.id, event.series.libraryId))