From 9243ab50d1dc83b9f7a2e2ab223fefdd455adb07 Mon Sep 17 00:00:00 2001 From: Gauthier Roebroeck Date: Tue, 13 Aug 2019 16:37:28 +0800 Subject: [PATCH] rework FileSystemScanner.kt to use java.nio instead of java.io --- komga/build.gradle.kts | 3 + .../org/gotson/komga/domain/model/Book.kt | 6 +- .../org/gotson/komga/domain/model/Serie.kt | 6 +- .../komga/domain/service/FileSystemScanner.kt | 35 +++++--- .../komga/domain/service/LibraryManager.kt | 27 ++++++ .../scheduler/RootScannerController.kt | 17 +--- .../domain/service/FileSystemScannerTest.kt | 86 +++++++++++++++++++ 7 files changed, 151 insertions(+), 29 deletions(-) create mode 100644 komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryManager.kt create mode 100644 komga/src/test/kotlin/org/gotson/komga/domain/service/FileSystemScannerTest.kt diff --git a/komga/build.gradle.kts b/komga/build.gradle.kts index 7a00221e8..89fea899e 100644 --- a/komga/build.gradle.kts +++ b/komga/build.gradle.kts @@ -52,6 +52,8 @@ dependencies { implementation("com.github.klinq:klinq-jpaspec:0.8") + implementation("commons-io:commons-io:2.6") + runtimeOnly("com.h2database:h2:1.4.199") testImplementation("org.springframework.boot:spring-boot-starter-test") { @@ -62,6 +64,7 @@ dependencies { testImplementation("org.junit.jupiter:junit-jupiter:5.5.0") testImplementation("com.ninja-squad:springmockk:1.1.2") testImplementation("io.mockk:mockk:1.9.3") + testImplementation("com.google.jimfs:jimfs:1.1") } tasks { diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/Book.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/Book.kt index 886a96135..9d591be76 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/Book.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/Book.kt @@ -20,4 +20,8 @@ data class Book( @ManyToOne(fetch = FetchType.EAGER) @JsonManagedReference var serie: Serie? = null -) \ No newline at end of file +) { + override fun toString(): String { + return "Book((id=$id, name=$name, url=$url)" + } +} \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/model/Serie.kt b/komga/src/main/kotlin/org/gotson/komga/domain/model/Serie.kt index 17719c2b0..c0c3b622b 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/model/Serie.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/model/Serie.kt @@ -21,4 +21,8 @@ data class Serie( @OneToMany(cascade = [CascadeType.ALL], fetch = FetchType.EAGER, mappedBy = "serie") @JsonBackReference val books: List = emptyList() -) \ No newline at end of file +) { + override fun toString(): String { + return "Serie(id=$id, name=$name, url=$url)" + } +} \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt index d0cf539bb..af1c0309a 100644 --- a/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/FileSystemScanner.kt @@ -1,34 +1,43 @@ package org.gotson.komga.domain.service import mu.KotlinLogging +import org.apache.commons.io.FilenameUtils import org.gotson.komga.domain.model.Book import org.gotson.komga.domain.model.Serie import org.springframework.stereotype.Service -import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import kotlin.streams.asSequence +import kotlin.streams.toList private val logger = KotlinLogging.logger {} @Service class FileSystemScanner( ) { - fun scanRootFolder(root: String): List { + + val supportedExtensions = listOf("cbr", "rar", "cbz", "zip") + + fun scanRootFolder(root: Path): List { logger.info { "Scanning folder: $root" } - return File(root).walk() - .filter { !it.isHidden } - .filter { it.isDirectory } - .filter { it.path != root } + + return Files.walk(root).asSequence() + .filter { !Files.isHidden(it) } + .filter { Files.isDirectory(it) } .mapNotNull { dir -> - val books = dir.listFiles { f -> f.isFile } - ?.map { + val books = Files.list(dir) + .filter { Files.isRegularFile(it) } + .filter { supportedExtensions.contains(FilenameUtils.getExtension(it.fileName.toString())) } + .map { Book( - name = it.nameWithoutExtension, - url = it.toURI().toURL() + name = FilenameUtils.getBaseName(it.fileName.toString()), + url = it.toUri().toURL() ) - } + }.toList() if (books.isNullOrEmpty()) return@mapNotNull null Serie( - name = dir.name, - url = dir.toURI().toURL(), + name = dir.fileName.toString(), + url = dir.toUri().toURL(), books = books ).also { serie -> serie.books.forEach { it.serie = serie } diff --git a/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryManager.kt b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryManager.kt new file mode 100644 index 000000000..1850d2f1b --- /dev/null +++ b/komga/src/main/kotlin/org/gotson/komga/domain/service/LibraryManager.kt @@ -0,0 +1,27 @@ +package org.gotson.komga.domain.service + +import mu.KotlinLogging +import org.gotson.komga.domain.persistence.SerieRepository +import org.gotson.komga.infrastructure.configuration.KomgaProperties +import org.springframework.stereotype.Service +import java.nio.file.Paths +import kotlin.system.measureTimeMillis + +private val logger = KotlinLogging.logger {} + +@Service +class LibraryManager( + private val komgaProperties: KomgaProperties, + private val fileSystemScanner: FileSystemScanner, + private val serieRepository: SerieRepository +) { + fun scanRootFolder() { + logger.info { "Scanning root folder: ${komgaProperties.rootFolder}" } + measureTimeMillis { + val series = fileSystemScanner.scanRootFolder(Paths.get(komgaProperties.rootFolder)) + + if (series.isNotEmpty()) + serieRepository.saveAll(series) + }.also { logger.info { "Scan finished in $it ms" } } + } +} \ No newline at end of file diff --git a/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/RootScannerController.kt b/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/RootScannerController.kt index 6e0cdb3df..bd17dd817 100644 --- a/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/RootScannerController.kt +++ b/komga/src/main/kotlin/org/gotson/komga/interfaces/scheduler/RootScannerController.kt @@ -1,33 +1,22 @@ package org.gotson.komga.interfaces.scheduler import mu.KotlinLogging -import org.gotson.komga.domain.persistence.SerieRepository -import org.gotson.komga.domain.service.FileSystemScanner -import org.gotson.komga.infrastructure.configuration.KomgaProperties +import org.gotson.komga.domain.service.LibraryManager import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.context.event.EventListener import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Controller -import kotlin.system.measureTimeMillis private val logger = KotlinLogging.logger {} @Controller class RootScannerController( - private val komgaProperties: KomgaProperties, - private val fileSystemScanner: FileSystemScanner, - private val serieRepository: SerieRepository + private val libraryManager: LibraryManager ) { @EventListener(ApplicationReadyEvent::class) @Scheduled(cron = "#{@komgaProperties.rootFolderScanCron ?: '-'}") fun scanRootFolder() { - logger.info { "Scanning root folder: ${komgaProperties.rootFolder}" } - measureTimeMillis { - val series = fileSystemScanner.scanRootFolder(komgaProperties.rootFolder) - - if (series.isNotEmpty()) - serieRepository.saveAll(series) - }.also { logger.info { "Scan finished in $it ms" } } + libraryManager.scanRootFolder() } } \ No newline at end of file diff --git a/komga/src/test/kotlin/org/gotson/komga/domain/service/FileSystemScannerTest.kt b/komga/src/test/kotlin/org/gotson/komga/domain/service/FileSystemScannerTest.kt new file mode 100644 index 000000000..a5d9fa964 --- /dev/null +++ b/komga/src/test/kotlin/org/gotson/komga/domain/service/FileSystemScannerTest.kt @@ -0,0 +1,86 @@ +package org.gotson.komga.domain.service + +import com.google.common.jimfs.Configuration +import com.google.common.jimfs.Jimfs +import org.apache.commons.io.FilenameUtils +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import java.nio.file.Files + +class FileSystemScannerTest { + + private val scanner = FileSystemScanner() + + @Test + fun `given empty root directory when scanning then return empty list`() { + Jimfs.newFileSystem(Configuration.unix()).use { fs -> + val root = fs.getPath("/root") + Files.createDirectory(root) + + val series = scanner.scanRootFolder(root) + + assertThat(series).isEmpty() + } + } + + @Test + fun `given root directory with only files when scanning then return 1 serie containing those files as books`() { + Jimfs.newFileSystem(Configuration.unix()).use { fs -> + val root = fs.getPath("/root") + Files.createDirectory(root) + + val files = listOf("file1.cbz", "file2.cbz") + files.forEach { Files.createFile(root.resolve(it)) } + + val series = scanner.scanRootFolder(root) + + assertThat(series).hasSize(1) + assertThat(series[0].books).hasSize(2) + assertThat(series[0].books.map { it.name }).containsExactlyInAnyOrderElementsOf(files.map { FilenameUtils.removeExtension(it) }) + } + } + + @Test + fun `given directory with unsupported files when scanning then return a serie excluding those files as books`() { + Jimfs.newFileSystem(Configuration.unix()).use { fs -> + val root = fs.getPath("/root") + Files.createDirectory(root) + + val files = listOf("file1.cbz", "file2.txt", "file3") + files.forEach { Files.createFile(root.resolve(it)) } + + val series = scanner.scanRootFolder(root) + + assertThat(series).hasSize(1) + assertThat(series[0].books).hasSize(1) + assertThat(series[0].books.map { it.name }).containsExactly("file1") + } + } + + @Test + fun `given directory with sub-directories containing files when scanning then return 1 serie per folder containing direct files as books`() { + Jimfs.newFileSystem(Configuration.unix()).use { fs -> + val root = fs.getPath("/root") + Files.createDirectory(root) + + val subDirs = listOf( + "serie1" to listOf("volume1.cbz", "volume2.cbz"), + "serie2" to listOf("book1.cbz", "book2.cbz") + ).toMap() + + subDirs.forEach { (dir, files) -> + Files.createDirectory(root.resolve(dir)) + files.forEach { Files.createFile(root.resolve(fs.getPath(dir, it))) } + } + + val series = scanner.scanRootFolder(root) + + assertThat(series).hasSize(2) + + assertThat(series.map { it.name }).containsExactlyInAnyOrderElementsOf(subDirs.keys) + series.forEach { serie -> + assertThat(serie.books.map { it.name }).containsExactlyInAnyOrderElementsOf(subDirs[serie.name]?.map { FilenameUtils.removeExtension(it) }) + } + } + } +} \ No newline at end of file