rework FileSystemScanner.kt to use java.nio instead of java.io

This commit is contained in:
Gauthier Roebroeck 2019-08-13 16:37:28 +08:00
parent b16d66a7b1
commit 9243ab50d1
7 changed files with 151 additions and 29 deletions

View file

@ -52,6 +52,8 @@ dependencies {
implementation("com.github.klinq:klinq-jpaspec:0.8")
implementation("commons-io:commons-io:2.6")
runtimeOnly("com.h2database:h2:1.4.199")
testImplementation("org.springframework.boot:spring-boot-starter-test") {
@ -62,6 +64,7 @@ dependencies {
testImplementation("org.junit.jupiter:junit-jupiter:5.5.0")
testImplementation("com.ninja-squad:springmockk:1.1.2")
testImplementation("io.mockk:mockk:1.9.3")
testImplementation("com.google.jimfs:jimfs:1.1")
}
tasks {

View file

@ -20,4 +20,8 @@ data class Book(
@ManyToOne(fetch = FetchType.EAGER)
@JsonManagedReference
var serie: Serie? = null
)
) {
override fun toString(): String {
return "Book((id=$id, name=$name, url=$url)"
}
}

View file

@ -21,4 +21,8 @@ data class Serie(
@OneToMany(cascade = [CascadeType.ALL], fetch = FetchType.EAGER, mappedBy = "serie")
@JsonBackReference
val books: List<Book> = emptyList()
)
) {
override fun toString(): String {
return "Serie(id=$id, name=$name, url=$url)"
}
}

View file

@ -1,34 +1,43 @@
package org.gotson.komga.domain.service
import mu.KotlinLogging
import org.apache.commons.io.FilenameUtils
import org.gotson.komga.domain.model.Book
import org.gotson.komga.domain.model.Serie
import org.springframework.stereotype.Service
import java.io.File
import java.nio.file.Files
import java.nio.file.Path
import kotlin.streams.asSequence
import kotlin.streams.toList
private val logger = KotlinLogging.logger {}
@Service
class FileSystemScanner(
) {
fun scanRootFolder(root: String): List<Serie> {
val supportedExtensions = listOf("cbr", "rar", "cbz", "zip")
fun scanRootFolder(root: Path): List<Serie> {
logger.info { "Scanning folder: $root" }
return File(root).walk()
.filter { !it.isHidden }
.filter { it.isDirectory }
.filter { it.path != root }
return Files.walk(root).asSequence()
.filter { !Files.isHidden(it) }
.filter { Files.isDirectory(it) }
.mapNotNull { dir ->
val books = dir.listFiles { f -> f.isFile }
?.map {
val books = Files.list(dir)
.filter { Files.isRegularFile(it) }
.filter { supportedExtensions.contains(FilenameUtils.getExtension(it.fileName.toString())) }
.map {
Book(
name = it.nameWithoutExtension,
url = it.toURI().toURL()
name = FilenameUtils.getBaseName(it.fileName.toString()),
url = it.toUri().toURL()
)
}
}.toList()
if (books.isNullOrEmpty()) return@mapNotNull null
Serie(
name = dir.name,
url = dir.toURI().toURL(),
name = dir.fileName.toString(),
url = dir.toUri().toURL(),
books = books
).also { serie ->
serie.books.forEach { it.serie = serie }

View file

@ -0,0 +1,27 @@
package org.gotson.komga.domain.service
import mu.KotlinLogging
import org.gotson.komga.domain.persistence.SerieRepository
import org.gotson.komga.infrastructure.configuration.KomgaProperties
import org.springframework.stereotype.Service
import java.nio.file.Paths
import kotlin.system.measureTimeMillis
private val logger = KotlinLogging.logger {}
@Service
class LibraryManager(
private val komgaProperties: KomgaProperties,
private val fileSystemScanner: FileSystemScanner,
private val serieRepository: SerieRepository
) {
fun scanRootFolder() {
logger.info { "Scanning root folder: ${komgaProperties.rootFolder}" }
measureTimeMillis {
val series = fileSystemScanner.scanRootFolder(Paths.get(komgaProperties.rootFolder))
if (series.isNotEmpty())
serieRepository.saveAll(series)
}.also { logger.info { "Scan finished in $it ms" } }
}
}

View file

@ -1,33 +1,22 @@
package org.gotson.komga.interfaces.scheduler
import mu.KotlinLogging
import org.gotson.komga.domain.persistence.SerieRepository
import org.gotson.komga.domain.service.FileSystemScanner
import org.gotson.komga.infrastructure.configuration.KomgaProperties
import org.gotson.komga.domain.service.LibraryManager
import org.springframework.boot.context.event.ApplicationReadyEvent
import org.springframework.context.event.EventListener
import org.springframework.scheduling.annotation.Scheduled
import org.springframework.stereotype.Controller
import kotlin.system.measureTimeMillis
private val logger = KotlinLogging.logger {}
@Controller
class RootScannerController(
private val komgaProperties: KomgaProperties,
private val fileSystemScanner: FileSystemScanner,
private val serieRepository: SerieRepository
private val libraryManager: LibraryManager
) {
@EventListener(ApplicationReadyEvent::class)
@Scheduled(cron = "#{@komgaProperties.rootFolderScanCron ?: '-'}")
fun scanRootFolder() {
logger.info { "Scanning root folder: ${komgaProperties.rootFolder}" }
measureTimeMillis {
val series = fileSystemScanner.scanRootFolder(komgaProperties.rootFolder)
if (series.isNotEmpty())
serieRepository.saveAll(series)
}.also { logger.info { "Scan finished in $it ms" } }
libraryManager.scanRootFolder()
}
}

View file

@ -0,0 +1,86 @@
package org.gotson.komga.domain.service
import com.google.common.jimfs.Configuration
import com.google.common.jimfs.Jimfs
import org.apache.commons.io.FilenameUtils
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import java.nio.file.Files
class FileSystemScannerTest {
private val scanner = FileSystemScanner()
@Test
fun `given empty root directory when scanning then return empty list`() {
Jimfs.newFileSystem(Configuration.unix()).use { fs ->
val root = fs.getPath("/root")
Files.createDirectory(root)
val series = scanner.scanRootFolder(root)
assertThat(series).isEmpty()
}
}
@Test
fun `given root directory with only files when scanning then return 1 serie containing those files as books`() {
Jimfs.newFileSystem(Configuration.unix()).use { fs ->
val root = fs.getPath("/root")
Files.createDirectory(root)
val files = listOf("file1.cbz", "file2.cbz")
files.forEach { Files.createFile(root.resolve(it)) }
val series = scanner.scanRootFolder(root)
assertThat(series).hasSize(1)
assertThat(series[0].books).hasSize(2)
assertThat(series[0].books.map { it.name }).containsExactlyInAnyOrderElementsOf(files.map { FilenameUtils.removeExtension(it) })
}
}
@Test
fun `given directory with unsupported files when scanning then return a serie excluding those files as books`() {
Jimfs.newFileSystem(Configuration.unix()).use { fs ->
val root = fs.getPath("/root")
Files.createDirectory(root)
val files = listOf("file1.cbz", "file2.txt", "file3")
files.forEach { Files.createFile(root.resolve(it)) }
val series = scanner.scanRootFolder(root)
assertThat(series).hasSize(1)
assertThat(series[0].books).hasSize(1)
assertThat(series[0].books.map { it.name }).containsExactly("file1")
}
}
@Test
fun `given directory with sub-directories containing files when scanning then return 1 serie per folder containing direct files as books`() {
Jimfs.newFileSystem(Configuration.unix()).use { fs ->
val root = fs.getPath("/root")
Files.createDirectory(root)
val subDirs = listOf(
"serie1" to listOf("volume1.cbz", "volume2.cbz"),
"serie2" to listOf("book1.cbz", "book2.cbz")
).toMap()
subDirs.forEach { (dir, files) ->
Files.createDirectory(root.resolve(dir))
files.forEach { Files.createFile(root.resolve(fs.getPath(dir, it))) }
}
val series = scanner.scanRootFolder(root)
assertThat(series).hasSize(2)
assertThat(series.map { it.name }).containsExactlyInAnyOrderElementsOf(subDirs.keys)
series.forEach { serie ->
assertThat(serie.books.map { it.name }).containsExactlyInAnyOrderElementsOf(subDirs[serie.name]?.map { FilenameUtils.removeExtension(it) })
}
}
}
}